{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 3 - Deep Q-learning\n",
    "### Deep Reinforcement Learning *in Action*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 3.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Gridworld import Gridworld\n",
    "game = Gridworld(size=4, mode='static')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.makeMove('d')\n",
    "game.makeMove('d')\n",
    "game.makeMove('l')\n",
    "game.display()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.reward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "game.board.render_np()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from Gridworld import Gridworld\n",
    "from IPython.display import clear_output\n",
    "import random\n",
    "from matplotlib import pylab as plt\n",
    "\n",
    "l1 = 64\n",
    "l2 = 150\n",
    "l3 = 100\n",
    "l4 = 4\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l3,l4)\n",
    ")\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "gamma = 0.9\n",
    "epsilon = 1.0\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "gamma = 0.9\n",
    "epsilon = 1.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 3.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_set = {\n",
    "    0: 'u',\n",
    "    1: 'd',\n",
    "    2: 'l',\n",
    "    3: 'r',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1000\n",
    "losses = [] #A\n",
    "for i in range(epochs): #B\n",
    "    game = Gridworld(size=4, mode='static') #C\n",
    "    state_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/10.0 #D\n",
    "    state = torch.from_numpy(state_).float() #E\n",
    "    status = 1 #F\n",
    "    while(status == 1): #G\n",
    "        qval = model(state1) #H\n",
    "        qval_ = qval.data.numpy()\n",
    "        if (random.random() < epsilon): #I\n",
    "            action_ = np.random.randint(0,4)\n",
    "        else:\n",
    "            action_ = np.argmax(qval_)\n",
    "        \n",
    "        action = action_set[action_] #J\n",
    "        game.makeMove(action) #K\n",
    "        state2_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/10.0\n",
    "        state2 = torch.from_numpy(state2_).float() #L\n",
    "        reward = game.reward()\n",
    "        with torch.no_grad():\n",
    "            newQ = model(state2.reshape(1,64))\n",
    "        maxQ = torch.max(newQ) #M\n",
    "        if reward == -1: #N\n",
    "            Y = reward + (gamma * maxQ)\n",
    "        else:\n",
    "            Y = reward\n",
    "        Y = torch.Tensor([Y]).detach()\n",
    "        X = qval.squeeze()[action_] #O\n",
    "        loss = loss_fn(X, Y) #P\n",
    "        print(i, loss.item())\n",
    "        clear_output(wait=True)\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        losses.append(loss.item())\n",
    "        optimizer.step()\n",
    "        state1 = state2\n",
    "        if reward != -1: #Q\n",
    "            status = 0\n",
    "    if epsilon > 0.1: #R\n",
    "        epsilon -= (1/epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.plot(losses)\n",
    "plt.xlabel(\"Epochs\",fontsize=22)\n",
    "plt.ylabel(\"Loss\",fontsize=22)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### PyTorch Automatic Differentiation Review"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = torch.Tensor([2.0])\n",
    "m.requires_grad=True\n",
    "b = torch.Tensor([1.0])\n",
    "b.requires_grad=True\n",
    "def linear_model(x,m,b):\n",
    "    y = m + b#@ x + b\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#with torch.no_grad():\n",
    "y = linear_model(torch.Tensor([4]),m,b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.grad_fn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "m.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 3.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model, mode='static', display=True):\n",
    "    i = 0\n",
    "    test_game = Gridworld(mode=mode)\n",
    "    state_ = test_game.board.render_np().reshape(1,64) + np.random.rand(1,64)/10.0\n",
    "    state = torch.from_numpy(state_).float()\n",
    "    if display:\n",
    "        print(\"Initial State:\")\n",
    "        print(test_game.display())\n",
    "    status = 1\n",
    "    while(status == 1): #A\n",
    "        qval = model(state)\n",
    "        qval_ = qval.data.numpy()\n",
    "        action_ = np.argmax(qval_) #B\n",
    "        action = action_set[action_]\n",
    "        if display:\n",
    "            print('Move #: %s; Taking action: %s' % (i, action))\n",
    "        test_game.makeMove(action)\n",
    "        state_ = test_game.board.render_np().reshape(1,64) + np.random.rand(1,64)/10.0\n",
    "        state = torch.from_numpy(state_).float()\n",
    "        if display:\n",
    "            print(test_game.display())\n",
    "        reward = test_game.reward()\n",
    "        if reward != -1:\n",
    "            if reward > 0:\n",
    "                status = 2\n",
    "                if display:\n",
    "                    print(\"Game won! Reward: %s\" % (reward,))\n",
    "            else:\n",
    "                status = 0\n",
    "                if display:\n",
    "                    print(\"Game LOST. Reward: %s\" % (reward,))\n",
    "        i += 1\n",
    "        if (i > 15):\n",
    "            if display:\n",
    "                print(\"Game lost; too many moves.\")\n",
    "            break\n",
    "    \n",
    "    win = True if status == 2 else False\n",
    "    return win"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_model(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Listing 3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "l1 = 64\n",
    "l2 = 150\n",
    "l3 = 100\n",
    "l4 = 4\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l3,l4)\n",
    ")\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "gamma = 0.9\n",
    "epsilon = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4999 0.06030523404479027\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "epochs = 5000\n",
    "losses = []\n",
    "mem_size = 1000 #A\n",
    "batch_size = 200 #B\n",
    "replay = deque(maxlen=mem_size) #C\n",
    "max_moves = 50 #D\n",
    "h = 0\n",
    "for i in range(epochs):\n",
    "    game = Gridworld(size=4, mode='random')\n",
    "    state1_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/100.0\n",
    "    state1 = torch.from_numpy(state1_).float()\n",
    "    status = 1\n",
    "    mov = 0\n",
    "    while(status == 1): \n",
    "        mov += 1\n",
    "        qval = model(state1) #E\n",
    "        qval_ = qval.data.numpy()\n",
    "        if (random.random() < epsilon): #F\n",
    "            action_ = np.random.randint(0,4)\n",
    "        else:\n",
    "            action_ = np.argmax(qval_)\n",
    "        \n",
    "        action = action_set[action_]\n",
    "        game.makeMove(action)\n",
    "        state2_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/100.0\n",
    "        state2 = torch.from_numpy(state2_).float()\n",
    "        reward = game.reward()\n",
    "        done = True if reward > 0 else False\n",
    "        exp =  (state1, action_, reward, state2, done) #G\n",
    "        replay.append(exp) #H\n",
    "        state1 = state2\n",
    "        \n",
    "        if len(replay) > batch_size: #I\n",
    "            minibatch = random.sample(replay, batch_size) #J\n",
    "            state1_batch = torch.cat([s1 for (s1,a,r,s2,d) in minibatch]) #K\n",
    "            action_batch = torch.Tensor([a for (s1,a,r,s2,d) in minibatch])\n",
    "            reward_batch = torch.Tensor([r for (s1,a,r,s2,d) in minibatch])\n",
    "            state2_batch = torch.cat([s2 for (s1,a,r,s2,d) in minibatch])\n",
    "            done_batch = torch.Tensor([d for (s1,a,r,s2,d) in minibatch])\n",
    "            \n",
    "            Q1 = model(state1_batch) #L\n",
    "            with torch.no_grad():\n",
    "                Q2 = model(state2_batch) #M\n",
    "            \n",
    "            Y = reward_batch + gamma * ((1 - done_batch) * torch.max(Q2,dim=1)[0]) #N\n",
    "            X = Q1.gather(dim=1,index=action_batch.long().unsqueeze(dim=1)).squeeze()\n",
    "            loss = loss_fn(X, Y.detach())\n",
    "            print(i, loss.item())\n",
    "            clear_output(wait=True)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "\n",
    "        if reward != -1 or mov > max_moves: #O\n",
    "            status = 0\n",
    "            mov = 0\n",
    "losses = np.array(losses)\n",
    "\n",
    "#A Set the total size of the experience replay memory\n",
    "#B Set the minibatch size\n",
    "#C Create the memory replay as a deque list\n",
    "#D Maximum number of moves before game is over\n",
    "#E Compute Q-values from input state in order to select action\n",
    "#F Select action using epsilon-greedy strategy\n",
    "#G Create experience of state, reward, action and next state as a tuple\n",
    "#H Add experience to experience replay list\n",
    "#I If replay list is at least as long as minibatch size, begin minibatch training\n",
    "#J Randomly sample a subset of the replay list\n",
    "#K Separate out the components of each experience into separate minibatch tensors\n",
    "#L Re-compute Q-values for minibatch of states to get gradients\n",
    "#M Compute Q-values for minibatch of next states but don't compute gradients\n",
    "#N Compute the target Q-values we want the DQN to learn\n",
    "#O If game is over, reset status and mov number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def running_mean(x,N=50):\n",
    "    c = x.shape[0] - N\n",
    "    y = np.zeros(c)\n",
    "    conv = np.ones(N)\n",
    "    for i in range(c):\n",
    "        y[i] = (x[i:i+N] @ conv)/N\n",
    "    return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmUAAAG3CAYAAAD1kSKeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XeYG9W5x/HfccOmN1NCMxB6QgmGkJheTUtIuYEUSAiBkBBCcpObmF4CwYHQAoReTTG9Glfcjdu6917X9q7brtder7ed+4dGa5WRNJJGmtHu9/M8fqwdzYyOjkbSq1PeY6y1AgAAQLDaBV0AAAAAEJQBAACEAkEZAABACBCUAQAAhABBGQAAQAgQlAEAAIQAQRkAAEAIEJQBAACEAEEZAABACHQIugC52HvvvW23bt2CLgYAAEBGkyZNWmet7Zppv5IMyrp166aysrKgiwEAAJCRMWaZl/3ovgQAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICjLoLa+UQ1NzUEXAwAAtHIEZRkce9dAXfPShKCLAQAAWjmCMg/GLl4fdBEAAEArR1AGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACoQnKjDHtjTFTjDGfB10WAACAYgtNUCbpFklzgi6EX96ZuFw/f3Fc0MUAAAAlIhRBmTHmQEmXSnox6LL45e8fzNCYheuDLgYAACgRoQjKJD0u6W+SmlPtYIy5wRhTZowpW7t2bfFKBgAAUASBB2XGmMskVVprJ6Xbz1r7vLW2u7W2e9euXYtUOgAAgOIIPCiT1EPS94wxSyX1lXSuMeaNYIsEAABQXIEHZdbaW621B1pru0m6StJQa+0vAi4WAABAUQUelAEAAEDqEHQBYllrh0saHnAxAAAAio6WMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCsjSamm3QRQAAAG0EQRkAAEAIEJQBAACEAEEZAABACBCUpWEtY8oAAEBxEJQBAACEAEEZAABACBCUAQAAhABBGQAAQAgQlKXBMH8AAFAsBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZWmQ0B8AABQLQRkAAEAIEJQBAACEAEEZAABACBCUAQAAhABBWRo2Jqf/yPlrAywJAABo7QjKPLrm5QlBFwEAALRiBGUAAAAhQFAGAAAQAgRlAAAAIUBQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlaRiZoIsAAADaCIIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCsjSsbNBFAAAAbQRBGQAAQAgQlAEAAIQAQRkAAEAIEJQBAACEAEEZAABACBCUAQAAhABBGQAAQAgQlIXEwsoaDZi5JuhiAACAgHQIugCIOP/RkZKkpb0vDbgkAAAgCLSUAQAAhABBGQAAQAgEHpQZYzobYyYYY6YZY2YZY+4NukwAAADFFoYxZdsknWut3WyM6ShptDGmv7V2XNAFAwAAKJbAgzJrrZW02fmzo/PPBlciAACA4gu8+1KSjDHtjTFTJVVKGmytHe+yzw3GmDJjTNnatWuLX0gAAIACCkVQZq1tstaeKOlASacaY77hss/z1tru1truXbt2LX4hAQAACigUQVmUtbZK0jBJPYMuCwAAQDEFHpQZY7oaY3Z3bneRdIGkucGWCgAAoLgCH+gvaX9Jrxlj2isSJL5rrf084DIBAAAUVeBBmbV2uqSTgi4HAABAkALvvgyziuptQRcBAAC0EQRlaWypbwy6CAAAoI0gKAMAAAgBgjIAAIAQICgDAAAIAYKyNCwrcAIAgCIhKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoS8OKkf4AAKA4CMrSYPYlAAAoFoIyAACAECAoK7DFazcHXQQAAFACCMoKbMOW+qCLAAAASgBBGQAAQAgQlAEAAIQAQRkAAEAIEJQBAACEAEEZAABACBCUpUHyWAAAUCwEZQVGXAcAALwgKAMAAAgBgjIAAIAQICgDAAAIAYKyNGwAI8L6jF1a9McEAADBIygLmTs/mRV0EQAAQAAIygAAAEKAoKzAyHUGAAC8ICgDAAAIAYKyNGjlAgAAxdLBrxMZY9pJ+rWkEyQtk/SctbbGr/MHobG5OegiAACANiLrljJjTC9jTK0x5uyEu/pJek7STZL+JWmsMWan/IsYnKFzK123r63ZpmdHLJKlKQ0AAPgkl+7LiyRtkjQiusEYc6GzvVzS/ZImSDpGkZazkpUq5vrzO1PVu/9czSzfVNwCAQCAViuXoOzrkmbb+GaiHymy9vZV1tq7JJ0raaOkn+VfxPAZvXCdJKm2vjHgkgAAgNYil6Bsb0mrE7adLmmNtfYrSbLWbpX0laRueZUuYJk6JzfW1helHAAAoPXLJShrltQyVswYs5ukoyWNSdivWtLuuRetdWDcGQAA8CKXoGyJpG87sy0l6TJJRtLohP26SlqXR9kAAADajFyCsk8l7SvpI2PMHyU9LKlJ0ifRHYwxRtJJigRwJYtGLgAAUCy5BGX/kjRH0uWSHpe0n6R/W2uXxexzuiItZYmtZwAAAHCRdfJYa221Maa7pB8r0mI20Vo7ImG3vSQ9Ialv/kUMjs041B8AAMAfOWX0d2ZX9klz/8eSPs61UKFBTAYAAIrE97UvjTF7GWPa+33eIBCTAQCAYsllmaUTjTF/M8YcnbD9QmPMCkmVktYaY673q5BBaGxq1gujFqfdh4kAAADAL7m0lN0s6Z+KLLUkSTLG7CvpQ0kHKNLAtLukZ4wxp/hRyCB8MHmlL0EXcRsAAPAil6Dsu5KmW2tXxWy7RtKOiszG7Czph865b867hAGprW/KuI8xRSgIAABoE3IJyvaRtCJh2/mSGiTda61tdAb6l0n6dp7lCwzxFgAAKKZcgrJdJG1O2HaqpMnW2uqYbYsU6c4sScanZrAvZiQuEwoAAJAsl6Bso6RDon8YY06UtJuS175sp0jrWUnyq2ty7poaf04EAABatVyCsjJF1r6Mdk3+WZHx7EMT9jtCUsk2E9F9CQAAiimXoOwJSe0lfWWMWS/pakmLJQ2M7mCM2VvSNyVN9aOQgfDUVEboBgAA/JF1UGatHSTp15KWSdpB0nBJl1trY6crXq1I4DY8/yIGw7dwi5wYAADAg1yXWXpV0qtpdnlW0stKnhBQMkh3AQAAiimnoCwTZ23MrYU4d7EYT21lNIMBAAB/5BWUGWMOkHSmtqe+KJc00lpbnm/BgkZLGQAAKKacgjJjzO6Snpb0EyWPS2s2xrwj6Q/W2qo8yxdyRG4AAMAfWQdlxpguiqS/OEGR/rtxisy+lKTDFMni/1NJxxhjTne6MksO4RYAACimXFrK/iTpRElfSbreWjsn9k5jzDGSnpPUQ9IfJf0r30IGge5LAABQTLnkKfuJIln9L00MyCTJ2fY9SVWSrsqveAAAAG1DLkHZEZKGJaxzGccZSzbM2bckWZ8mVlpmaAIAAA9yCcoAAADgs1yCsoWSzjbG7JJqB2PMrpLOdvYFAABABrkEZe9J2lPSp8aYryfe6Wz7SNIekt7Nr3ilz1sSWgAA0NblMvvyMUlXSjpL0hxjzDhJSxRJj3GYpNMUWfdyhqTHfSpnSDFeDAAA+CProMxaW2uMOUfSM5J+pEjqix6xu0h6X9LvrLW1vpSyhE1YuiHoIgAAgBKQ64Lk6yX9xBhzsKQzFL/M0ihr7XJjzL7GmIOttct9KmvgNm6p165dOsZsoWsSAAD4I6+1L52A680Ud38s6ZR8HyNMTvrHYP3m9ENb/ibBLAAA8EuhU2JkDFuMMQcZY4YZY2YbY2YZY24pcJnyMmDWmqCLAAAAWqEwtGI1SvqLtXayk2ZjkjFmsLV2dtAFAwAAKJbAk8daa1dbayc7t2skzdH2MWqBYV4lAAAopsCDsljGmG6STpI0PtiSAAAAFFdogjJjzM6SPpD0J2vtJpf7bzDGlBljytauXVv8Aqbx1NAFGj6vMuhiAACAEhaKoMwY01GRgOxNa+2HbvtYa5+31na31nbv2rVrcQsYV47kbf8eNF+/emVi8QsDAABajYwD/Y0xZ+Z47l297GSMMZJekjTHWvtojo/lOy/ZLtwCNETUNTSpoalZu3TumHlnAADgafblcOU27t14PK6HpKslzTDGTHW23Wat/SKHx/QN8VZ+ej4+UkvX12pp70uDLgoAACXBS1C2XAWMUay1o1WiqfFJHpva0vVtfoUtAACykjEos9Z2K0I5AAAA2rRQDPRHel/OqdDdn8wMuhgAAKCACMpKwHWvlem1sct8O98rY5boj29P8e18AAAgfwRlKbTmmZX3fjZbn05bFXQxAABADIIyAACAECAoAwAACAGCshS8pLsgIwYAAPALQRkAAEAIEJSl0JoH+gMAgPAhKMuSLUK09tWideozzr8UGAAAIPy8LLOEIvvZC+MlSVefdkjAJQEAAMVCSxkAAEAIEJQBAACEAEFZllZV12V9zC19WdIIAACkR1BWBJ9MZUkjAACQHkEZAABACBCUpVBb3xh0EQAAQBtCUJbCtsbmoIsAAADaEIKyPKRKI7uqamtRywEAAEofQVkBfLf30KCLAAAASgxBGQAAQAgQlAEAAIQAQVkKxgRdAgAA0JYQlKVgU43ij0HcBgAA/EJQBgAAEAIEZQAAACFAUAYAABACBGUAAAAhQFCWh5Ubt+r1sUuDLgYAAGgFOgRdgFJ23+ezgy5Ckj/1naIzjuiqH518YNBFAQAAWaClrJX5eOoq/eW9aUEXAwAAZImgLAWSxwIAgGIiKCui6SurNHfNpqCLAQAAQogxZSmYAuTr/95TYyRJS3tf6vu5AQBAaaOlLAUrD+ssAQAA+ISgDAAAIAQIykrIorWbgy4CAAAoEIKyEvKTZ8cGXQQAAFAgBGUpFGKgf77qGpqCLgIAACgQgjIAAIAQICgDAAAIAYKyAExatkHlVVuDLkYglq3foinLNwZdDAAAQoegLAA/emasTv/X0JyPn19RozEL1/lYouI56+Hh+sF/vwq6GAAAhA4Z/QNic8xNW1VbrwsfGymJlQEAAGhNaCkrIcYY3fz2lKCLAQAACoCgLAW/l1maX1Hjy3kqNtXldfy2RtJqAAAQRgRlRRLtcszH5m2NeZ9j2fravM8BAAD8R1AWcssJogAAaBMIylIIS0b/lVUEZQAAtAUEZQAAACFAUNbGhKP9DwAAJCIoKzHzKzYHXQQAAFAABGUAAAAhQFAGAAAQAgRlbYxhUBkAAKFEUAYAABACBGUp0KJUWH3GLdO8Nf4sPQUAQGtAUNZKbdxSH3QR0rrz45m66HFvS0+t3Fgra/1dixQAgLAhKGullm1ItRJAaTUBzltTo9P/NUwvjV4SdFEAACgogrI2bN3mbXkdP2bhuoK3yC1bv0WSNG7xhoI+Tphs2daouWs2BV0MAECREZSF3Mj563w5T2NTc1IXYFOz9y7B5marTXUNLX9va2zSz18cr1++MsGX8mG73/aZpJ6Pj1JDU3PQRQEAFBFBWYhV1dbr2RGL8j5PdW2Dvn57fz07YnHc9uYsxmk9PmS+jr9nkDY4LWPNTrwwv4LB+n6bsCTSKpjN6wMAKH0EZSFW35h7S0nsyLG1m+skSe9PWhE3qzSb7/x+M1ZLkjZsya/LEwAAuCMoC7Msx+TPLK/Oav982mFsXkcDAIBEBGUp5DpHsWJTna/lyMZlT47Oav980kxEu9hM0WZzEgQCAFq3DkEXoDWZtGyjxi1e79v58gl4Yrsp3xy/3HWffIYs/eqVibkfDAAAkhCU+ehHz3wVdBFcvTJmacvt0spSFqt0Sw4AgBd0X7YhJmHtKD8m97EcFQAA/gg8KDPGvGyMqTTGzAy6LGGTT8Djpeszm8H6QWdnGDKnQj97YVywhQAAoIACD8okvSqpZ9CFSBSGYeVecoA1NDWr2WMS2HySxy5eF8msv2LjVs/H+O2rRf6N1wMAIGwCD8qstSMltZ01dBLUNTTp1THu6zr+7IXxGY8/4vb++mPfKZ4fL7YLc+CsCs/HRa2qig/Kwtp7+djg+brg0RFBFwMAAM8CD8rCaEFFjZY6LUOF9sSXC3TPZ7NzOjaaXPbz6as97Z84pswtOe3TwxamzXdW7G7MxDJ79cSXC7SgcrPPpQEAoHBKJigzxtxgjCkzxpStXbu2oI9189tT1HfiioI+RlRVbUPmnVI48o7+Ke/LdTzawwPnpc13VuxuXZYaAgC0FSUTlFlrn7fWdrfWdu/atWtBH6t9u7B2yuVn3eZtqq1v9LRvt179NHVFVfIdRQ6S5q9hbU0AQNtQMkFZMbVrpXkeqmob9IOnvedSGz6vMmlbYkiWa/ciAACIF3hQZox5W9JYSUcZY1YaY64LukztitpSVpiWp/Kqra6zMuubto8jy2X9SnoTAQAojMCDMmvtT621+1trO1prD7TWvhR0mdq3gsaf3/aZpKeGLcz7PKc+MEQvjlrc8nc+62Xm4rPpq4r6eAAABCXwoCyMWkv35dgMeb28xFeVNdt0f785249JuL/QA/FXBpgXDQCAYiIoc1HcoKy0AsDEGKy2vqmgj1fo8wMAEBYEZS5aSUNZRicctFva+92WamJIWfEwfg8A2haCMhfFaik786Fh2rLNW4qKXGR6Grt27pj1OYs9pqxNaiM/CgAA8ToEXYAwalekUHX5hlp1aA2zCtJYtn6LDtlrp6CLAQBA6NFS5qK1DPTP9DRyafPKdmB/NoueA0AmKzbUqqYu95VQgDAjKHNRzISolZu2FezcbmPCsjre5fB0Mdl6lxUDVlXV5VUG5GfQrDWq3MRrgNbjjIeG6cfPjA26GEBBEJS5KGbu2M0FHFNWWeP/l3G6dq+T7x+iY+8aGLdtfgXLJAWlsalZN/SZpKueHxd0UQBfzYv5XKmqrdeaan54oHUgKHPRWrov51dsTnt/LmP2sz1my7ZG/c+zX2nJui0p9/lg0koNmLkm+8IgrehLtXxDbaDlAArptAe/1GkPfhl0MQBfEJS5aKXrkfsi26WZhs2r1MSlG3XR4yNT7vOX96bpxjcmeTrfmIXr9Ob4ZVmVIVZNXYNmlldndcyD/eeoW69+OT9m1nwahsdEWbQFdQ3NmXcCSgRBmYvWtMh2RZrxRLmkt1hVtVV3fjzT8/7Ruqxv9OeD8+cvjtftH3l/fEkaMHO1qmsjA4OvfWWiLntydFbP/bkRizPvVAB+XYZhuZznrtmk8ipWaACAVAjKXLSmlrLqrbnPUnKrhjfGLVefcd5bqoKuylVVW3XjG5N101uTJUllyzYGXKK2q+fjo9Sj99Cgi9Fq3f7RDJ1436CgiwEgD+Qpc9FaxpRJ0sYt9Z73/Wxa/OLfDa0gncU2p4Vu5cb4cVXWhqcFCfDDm+OXB10EAHmipcxFawrKrsxi5t3Nb0+J+7uq1ntAl0o2Vbm2ZptWVxe2e6sVvbQZZTv+r1g+mVqu33kcQwgUUk1dg5atTz0JCSg2WspctKUv7kLLJlfaKQ8MkSRNvesC7b5jJ1/LEQ1Pwj743Vqr+iZ/By7nm6/Ob7f0nRp0EQBJ0o+e+UrzKzZrae9LfT3vproGdenYXh3b0+6B7HDFuGhNLWXpZIpPtmxrKko5Ek1ZUVXwxwhrbDZ5eeGfO4CITGmDcnX8PYP02z60BiN7BGUuWtNA/3x8MHllMA9cgIgp+pKGJd621mr5+uT8YXUNwQTCfrHW6r2yFdrWWNrPwy/WWvWbvprlxtqgoXMrgy4CShBBmYtSbClbEdYEoSGpysSvxFzSgXi1qmprXC60n784TsffE7/SQd+JK3Tmw8M0admGnB6j2cOXfBBdtYNmV+j/3p+uRwfPL/6Dh9CHk8t101uT9cqYJUEXpaTV1DWU/A8WwAuCMhelmKds9upNWR9TjC/tXGrSzwHqQbyS3+09VJc9Obrl7zEL12tTXfxyWpOc1ByL1sYPMvZS3i9mrNZht32hRWs9dr0UsRKmOV3Pa2sKt6Zr2DU3W01eHnl9122O1ENlK6qPmeXVWr+5uM/nm/cMintPAa0VQZkLui9bv3zCvpnl1Z5aqtKZk0MQHdXfWZIq25UJiuG/wxdJ8taS11o9M2KRfvjfrzR+8fqgi1IQlz05Whc/Maroj7uwsjDjv9qqFRtqC7I+MvJDUOaiBBvKclvHsgjD3cNWl5mKU13boC1pFomfsGSDLntytF4anV931KxVKYKyhAJ+Om2VuvXqp011uScB9rqawqK1mzV3Te7BYqzEmCwsY8zqGprU8/GRLS2VhTBvTWSx7DUxq2kUsrtckuuC3FW19RpWoHFNranlLx8bttS7jg0tBWc8NEynPsCaoWFDUOaiFMeUhVUu69IFmbbihPsG6cyHhqW8P5qENpfuYjcbMiT3fdZpecrlgz/bejzvkRHq+bg/LSDNCQ9+/qMjsj7HFzNWa3OaADkXs1dv0tw1NfrH57N9PW+s2GderI+SX748IWnbDX0m6dpXJ2aVQBrZ6dF7qM58OPXnBZAtgjIXpTimLKz8WvPSb+kClvUpvsRWFWDdxinL41tsYnOKxa46UFXboEYnf1kpXJ2J9btiQ3Z1N3fNJv3+zcn6+wfTJUkLKmr07IhFeZcr+oMrm5ar618v09F39vccIEbPfUvfqUW7/t26oRY7Yw4bmsP5HkzHWqunhy0MfffaViYfwGcEZS5KcUxZWGdf5hLfRr8v03Uj5nrOfBQiKBs4q0IXPzEq43iZX7w0Xnd+Miurc/vRPV1etdX7hIIY+aaAiObIi9b5FU+PUe/+c1sC01xFL8dsijd4doXqGpp1gcfWvthTzytQHqyt9U0699/D045bi17zYUse7MX0ldV6eOA8/fkdEg3Df9FhIdW1uQ8LKRSCMhel2H0Z5C+2D9PkM8ulKn/zepkk5T1uK/bxlycErV4ClspNdXpz/PbF1wvVqzpn9Sbd9/lsLazcnLa+Pk9YmzSfxea96tF7qM57JPuux8Tuy3z5dX23tJTl8Gqudhm35ZXfXfLzK2q0eN0WPfDFnNSP6fxfgh9nanSi5tp6WqIgjVqwVt169fNt/N6LoxZLkpaGcIktgjIXpdhSlkuRl6zbov8OX6gZK/ObxTc9z+NTyaW1xVqrkfPXxvwdf3+qrulhcys1fWV8Nv3rXy/T7R/NzLoMuRg5f23SuKvEskf/3OisSXpXhpazIMfmRV+6cx8ZriufG5vDGeIL79eQguhpMvXoVdc26NFB83Jr8Ys5JFrqF334geH6UGmKF+1GLcGPMyDOB5MiP/wnLc8tr2MpIShz0VbGlN3+0Uw9NGCeLn8qv/w/6b64Zpb7MyDeq3fLVugal0HPiRK/zK59daK+99SYuG0bE5q2Y4+ZsGSDbn57iu+z6mKvvJEL1sa1ckQfK6zj9GJFy7p47RaNX5L7B2niOzHf2o62lGVqyftHv9n6z9CFGjRrTdaPEZZZzdtbysL5eba1vkn/Hjgv7czcsK9Vm6uZ5dXq1qsfi6F7VO4MY9hc58+QljBfVwRlLtymlyO1pgJd4bl8l6zcGD/uK5/vo/YJTaZlMdn3y6u26rNpqwqaJLUhx/FT37xnoM57ZHig63v6fU1EX4l8T9uunbfzRLtLG3Jqrc36kJxZWTU3W9d1aqPlCGvL/zMjFumpYQvVZ+yypPtCGkf65r2yFZJUsJQlrc3EpZEJUf1mrPb1vGG8zgjKXAzI4ddx0IK8uEJ4XfsisU4fGjAvaZ91m1OnG/j6bV/o4ynl+ZUhdjamx2Nq6hqTVgqQIi0TYxe5Dwx/a/zyXIqXkt9rPfp1fXttKcv0cDe8XqYevYf6U6gcxF4Xjwyep3qXAH5792U436HRFjK3srcVIW6wycrGLfV5D4Pxwq/x3sVozc4VQRnapKeGLkx7/8j5az19laX7jGhstvrXgLnZFSxGU7PVDJes/V4/ThK7Vm//aIZ++sI41y6T2z6akfZcb41fntXag7m0Fq3YUNvyGKmOz/fDNNpq5HUiQqru6UGzK1q6VJKP2X57XIGz+lsrfTJ1lft90RtFismWr6/NKkmwl2AxvF+d+YntUq7cVKduvfrpnYn+/jAqpp88NzblMJi6hibfVrfwu/EhjD9YCMqQt0J9cBbyDfPUsIWqa2jS7BSZ9a95eYJra1Pid3Quv9zS5buK/bAetzh+LNb2FAe5metkmq/xOC7j05jZnrd9NEMPD0xuKUwl2+CpudnqjIeG6Q9vTY7f7nNaB9OSp8zbfrmI7XZ2y3xfvbUh567pKE9jyqJ1V4TvnZq6Bp358DDd+kH64N6N22sRvq/KwrBWWrwu8jnzweRyfTK1XDV5rN4RlAVpUvrc/cksXfn8uJxS6yTyq6UsOtEnjHVNUNZKhHUwb6x8lgryQ2IKib++N02X/GdUVhnPt9TnP9A0MWFsrJ/EzFTMZv3Ibr366cY+k+K25Rss3/dZfNZ7t9UH1tZs82UJpWhZhzpjbKKTGaauiJ8Rm/eYMp+6L9P5MsM4oRPuHaTfvTE57T5ezVq1KWkcZaJifDJsdVJXjFq4zvMxsR9ZL49ekvX4zM+mubcQFsuSddt/tEUVMNh6AAAgAElEQVRX+pDkaaJRlJXU3xknNWf1Jt3Sd6p6fZh9YJvJtsYm34cUeDW3IvJjcJMPKXz8+p6LrshyXwFX9sgVQRnyZiQNmV2Rcb/j7xlUlLKksrp6a9z9g50yJ37xp+PHIO5cz5Gp9SlxLOS37huc2wPFPGImpzwwRDe/NSX5yCyf42jnyzz6vbEtcYap88Lluyi11+Sx67dEAoTFLq2lfhgyJ/P7JV8tY8p8/MG2NUXesFwu6WipFlTU6L7PZ+umt1wC1TQXUmIdfrXIe0CYyrC5lVpd7S1J9Dn/Ht5yO3aiRWxKnvKqrZq4NPXs4+HzKvWaM9Ehmix7SQGuuaPuGKBfvDje9/MWm9+TVrz2GhQTQVkrUegFj9M+tqRVHj/Iii1x2aJY0S/+Z7JYviexnovZQJntS9xYwF/Gm+oa9MLISALGQR4C8kzmZFhLNFrNlz2ZX/qW6OuVKcAdszAyBublAuUXy1e66y4auLakxPDxcf/v/Wmu2/PpWm9oihycb0vKz17IP+i49tWJuvzJMZl3TJDqejrroWH6n2dT5+mLbX2OnsGvdXUTjS3w+MZiKMXE7tkiKIMvcn2r/PT5ca7bF6/drHkV+X84xX5Yxq4lGWtCHnm0Enn5Nep5oH7Aw5yTEu/G3L77k1lps8l7sW7z9u6qxJelUF0tYRzY67dot1ohxpRNW5m+VTmbx4rum3ZN0QC+hGOvy3xl+mGU7jr/xYvj1ctZ+7VUPD1soQbMLFz2gtb/7iUog19iPjyzmaWX6tfbuY+M0Bcz8n9zV2yK/4AtdIPi6IQxNfkszZMYQCR16WV7viw/0RKrKvbvTK0aXuo51Zi5dZu3tSy1FeXXd/Og2dldUzUuAcPkNGMCw+DhgZEZv81FTImRyw+I6EogI2K6+5JPnPq8YUoAmm1ZotdzukTQoxeuU9+JK/IoVfE9PHCebnxjkut9frxcfsfofi8H5weCslYiTH3jR985IOgitEhcOqmUrE3xiz3dB1O69TCz/fwJqkv8oTzSiIxdtD5tbrj7++XXuidJP/zvV3mfo5DmVyR0XyasCpHP67piQ/o0INEAcEFFTcbWzi0eZyGXgmyrdKnTmrl4XevP6O/nK0n3JUrGc874nqCUwltlY219oEl2s5lQIKUeVJ2On10vid+p0ZQaUnKXU2NTs/qMi128Pfcv/nfLkhe499ra89MXxulP70zNuF8IfyCn9e7EFRoT0wrrqT5cnuOht36hKwocVC6srNEFj43UY4Pnp90v3y/Y+RU1Sdtyvf4Tr+dCz+xc4hKMFeKa/HJOheusaT/UNTTprfHLi/rjLfaaGTavUp9MzS85dz49GYVCUIY2Y2gBljTJ5mvliqcjA4i9foilCmzSHe5nzJnY6hYdjD9+8fqk9SxfG7tMd35cnMXbvahraNLf35+u9T4GqX5ozDE/2d8+mK6fe5w9F52hFr1+EoOfaVn+OMhWdMhA7LJkbrJZv9NN7I+EqIk5jg9NTMdx89vJM4rTyfZHSDHCmOqtDbrutTJd99rEgpz/scHzddtHM9S/gGPIEsVeM9e+MlG39M38A6zUEJTBF8VugerWq1/StjfGLUtKuBrL2u0zvUpBoX6A5ptXbPLy5C/1xACu2UPs4ZZY1S+fTC3XO2UrPK+oUNfQpDfGLcsqN1wubujjPt4mW+neb9EgrJjJY2O7SofPi/z4SfdelNK39kXvyScFSlOz1e0fzYhbweK1r5ZqVYqVGMLkR8/k35oZTVC8bH1thj1Tm7x8o+a5BL+StN5pgfO6SPigWRV6e0J+qxbQfQl4FMSstoufGBWXiPWOj2dqQkJOoKra7cGCH7MZk7Ls533G1BJbo7xINxYnOtX+yufcZ7x6f4zM+0xYukGzVqVfC++yJ0er2nl9MtVjrp/FXgPbx4bM1x0fz9Q7ZStSrvLgB7fW2vrGZj3Qb7ZvyZW3p/1w/s7xPM+PXJR17i8jpVx+Kmlfl4JNWrYhLjCuTejCr66Nr6N0y/fMLK/Wm+OXt7R6VdbU6e5PZ+lXryQnd8330yv78Zrp75+0LP/JJPmuACJFxk9e9PhI1/ui5/X6ufrsiEW6Nd/EuM6D9hm7NL/zhBhBGSQp72VfgjBn9Sb9IMP4mKeGbV/j0o+Wp8QErRc+NjKrMRXrN2/Le8ZPugDFywdwvvnL5rrkUXJ7XLcWtUTnPjLc02Nm+8US/ZHg9ZlGV3W49cMZuuQ/o7J8tNzc99lsLays0UdTVuqFUUv0iIdlrIbNq9SAmavTXwMJqxbkOmj+n1/M9Zz7K/oeWJVmjM5jg+erW69+LUFXYrHmrqnRj54ZqxdHpx4fe8J98Qmo043TbO/040YnHERbb6tqg19ap5jpbgrVuOT1vNmOpU2nnTFasaFWd34yy7dzhg1BGSRJ93ya+0W+prpO/Weu9rE0hVHIAakLKzfr7IeHZdzv5PuH6IE8ZwCmHVOWxQfwmuo6TysxJKr3GsB7qO9oF8jqDK0rW1JMeqiqrXcfpxVtLUq1sHlSDrb8vrn6z1itQ2/tp9osluF6ecwS/fLliap3utQbm63Wb96mhZXu3UVSZBzNjW9MTlteL1nPp6+s0tC5/q0qEFufqeo8+gOpKUOqjoWVmz1fx+m6sxKDssQWxFi9+6fu5v7Plwtch0vEyvajJdUsVi+mr6zy9FlW6MAv2vI+cn7+Kyl4ZVTYpNhhQFAGSfllex46t1KjFhTvjZmrQr2XrY109Sz1OHbDbaHzIPzgv2OS8oF54bXlJZvq9lp3seoamnTifYN1t8sPikxdK+VVW+NnM3oMAganCGL/PWierFXW45ViU1QYE8nPd/6j7t1FsdKNC0wcU+bme0+N0a9fjX/tm5qths2rLNpsOj9acBLPUbZso5qarZqbrS5+ItLiGf0Sb7kmXJ5eYgt4rEczzCItpsGzK/S9p8bo3s+yWbOxME1lM8sjQdngFEuGVWyq0y9TrANa19CUNkdbKn4vsxRGBGWIKOEfH1u2NQb+y7GYY+piv4i8tsy4pdfIdTq42zPN9ws2m1cmOq4omqT4zfHJg4dNumYRh9fZjLH+lrDMUENTs6q3NngKtD+fnpxmwWp7kNDOmLR55mJ9lCYXW2L3uNeX5qXRi3XtKxM1cJb7l+wAj63hqVo1o6LlS9fKlWtc+NLoJXriywVxrSkt+dJaHi64D7sR89fqkidy7x6/3vkR9epXS1tynaXk8jQzJfbeuKVeR97eP2mVkxUb0vxoStUyOnRhysTAR985QOc/OiJtWdy0M61/TQ6CMkgq7SSGx909MC5HVirVWwuTYNdK+rIA6TbcJOZPqmuI/7WZ6iNrzpr4cWCxg6nHLsquldTrDKhCNbgkjityk/UYNI8HJK1yYCNLgkXNSjNJwK2VzdrYzPvebUpzLSdeE14td754b3xjkmsX6l/f87bkT32G2b1+zgp9yGUc3sLKmrgfYI3OYLKWcYYFuC69/uD75csTcl7bsv+M+KB4Y623/GOx9bx0ffrP+UnLNqq+qVnPJawHnEty8lSvbzRp8PJ0gV7Kk2Z/SKkhKEOrkKpbKVahEkJaa31N2prOmIXpu4m9ftEddtsXLbd/+kJ2szE9BzAhyNC6sbZe3Xr103tlmZar8fakElNmJH4Zp8ub5FYdzdbGBCnev3Gyqdt0p12xobalKzT2lBc+ltyF6rV0mdoytjU2Z5xY5LUq3LrAlqyrVV399u3RAf4eGk/TSlfn1kZy0L0/KTnxsV9+9+bkuL+NMXp34grXFlgpv/bAbI5NFZCmeglnlKeflV3X0JQyn197UmIApaHUlmXJVbO1mrxs+2wmz4lofYyP3Go6Or4kV+meR6qVDdLlnIteDtEcTV5aUr3YlNBi8KuXvSfmdJt1a5Xbl2c2M3inpJn9dsZDw3TUHQO0JqEr23X8pcsLv60x8gUaW5xUb8Xoa3zCvYN04WMjPbe4Zjv2aM7qTTrv0eFJ27ePKcvtzZDusGZr9epXS/XX96al3slnRpGkwn94a4qejpllHpVtSoy6hiY98eUCSZHxYF6lqpdUn8mZXvej7xyQ8oeiMf7PJg3Dj8dYBGVoFYIMyYr5ln63bGXa2Y+pPrBWV/uXMNNt9tMQl8G+ftVLTYr8Xfd9nnqwc76tIl5lM0HGrSzbGppy+lJYn8XSOdHxQT95bmzKff749pSMC967fZkedccA/c9zY+NaS7w8nSXrtqS8VhNb2vpOzD7h6LrN2+tn+0SK/D4l0j2tAbPWZPWa+OHF0Utabj+cJp2K16d99J0DWlqxErvhP5i8Ur0+8NZ9nYmXHxQTl7rnaStE8tjXx/rzg80vBGVoFdrCrBw3Xr/O//CW92VjMi0FNNmHxJbZOPWfXyZt+/5To9N2R28fP+SthtKlocgkUzATtdhlMsCmusaWdRBf/Wppy/bYCRyL1iZntU9MqppOtAoSB2/Hqmtsyvhlmeo9NmV5VdzsWa/jq4bPcx8ELsUHZnflmZMqVWl69B6a3Xmc+lm2fkvS9bK1vqnoPwwzDcdwex2slV4evUSVNdlN8nlp9BL1neg+BCBV/aaKn9rn8WFdiB4Rr2PzioWgDK1CLikV/BKm1m8/PrT+MzS5KyRWugShsVLNvHKTbbmnrUw/LiXblrJUv8y9uOp5b2Py5qQY4J247qIUf0390WUdxmxa16q2Zv7SaWq2GSsr3WuUKvVBLt4pW+HrTOnEqtrozN5NXHkgU51G7z3r4eFJaUsyzWoMUmyAu3jtFt33+Wz94c3s1vZMpylFrqFUYwu9tnZ94+6BLsdmPq6+sVm3fjhdlR67YMO2dBNBGVqFJQHOHh2YJsdRoRUiIPSrLtO1hMRaU13ny7gOtxQBmU6bTbLXYspUG9nk3POyNmGz3Z7UNRWvDRyxS5HFdptnU+ZUX/S5iAZ4mYqfzw+7pub0Y9/yXW82F24vZ3T5pvVbCj8xKVWsMzvD8mtRm7clX7de4qcv51To7QkrPLewhiskIygD8nazS0tGUPz4gMl3GahsnfZgcvdkLp5NmMYvbZ92Pz1Fy1oYFqd2q+0tMV9Ibmk2vLYCpDp/0j7Wasu2TIFD9lfXsLmRwHxjluOt/LwGo6fKlIIhMRBMbP1K1xpmrU272HY+SztV1dbHLaruxY19JrWs0hIbyMx1UuMUI4F1qqsl3yWS3Frglq+vbZmF2pKWzuPlmunHSLERlHnU87j9gi4CkCTxi8KPlviq2nrd/clM1/vKlma/SLoXbuOtsjFhyYaUY16i0i1eHSS3PHE9Hx/Z8gXqxmsXsuStNXXumhrXyRqxcrm2osGV16S4UX4uxRtJ0Gv1/afHeNhzu6PvHKDXYsb5XfbkaL053n1QeLO1akjTupdPkHneIyN01sPDM+4X29o8YNYaDXJJE+R1/KMfCtEr+MY498D34idGtoybjbaMBp1PMVcEZR7t2Kl90EUAkpz3SHxW7O88mN3gZTdjFq7XaylmJP342dQz+PKROL4nW4kzCze5BAFXehz7VWxu3TQbaxv0vacyBRHezCj3Z0HoXMZn5xoETFjiXwBtbeSazkXsEl7L1tfq9o/cf6x8PHVV2u7LfHpjvc7qTJmaIuZ2vutGZtPdP68ieYJKocSuItHyFGOeeMWmupSzuEMWk6lD0AUoFWF74QApi8XB25j+M72N83tq6ELttfMOBS5NbjIlWPUq3xxyUiTJbMWm7MchvVe2QtedfmjW4xT/Pci/9SattbqhT+Y1XgvZYpKYdLggj2Gt2rl07cVO0NiWxYSEez9L7mbMJqgbmcVEn2xkagBrSYESs+3bzgzupb0vdTvAp5L5g6AMQKvzlceloz6eWphVHvwQpu+K3v3n5nRcfWOz6hubde2r3pPs+s1rS9Mzw5PHJPqlGK+ll3hp7prk1C8zVlbrmwfulrT9lTFLk7aFbVC8JE1LkRy5qdlqa32TumTo5cq39dBvdF96FLasvwBav3P+PTyn47YkdIkuzzNlzPgcuxObrfW0BFoYfJhmkfd8nfnwMHXr1a9g55dS54fLNDTg8qdGa3aaNVsLqVuvfnk/9geT45e2in5V95+5RsfcNSDj8f8tYDCeC4Iyj9wu9z7XnapTu+1Z9LIAaBtyTU9yXEKOpzMfHpZXOWIz5Efd8fGMjMc128IM+EayfNoNvCaTNcaoudnqwf5zcn+wBGXLIpOHVlVt1W89dDMnzqSOTUZrrU2aVFGsdYn9QvelRzvtkFxVZxzRVWcc0VUzVlbrgD266Fv/GBxAydDWzcywwC9QCKlmwsVavqE2dMk5W6srnh6juWtq3MdNZZBNPDdh6QY9N2Jx2n2yyTPX6Kxh+68BczVwVnyrqrU2KWnxTW/FL8y+IGZCgbXJwalb/sIwo6XMg+euPlm3XXKM/u+io/TkT09Kuv+bB+6mXTv7G9/eeNbhvp4PrddlT44OughASn5NWEB60fFiuQy1+d0bkzztZ+RtRq3b7OdUHh+SelLHP7+Yk3G1hNEL17XcbrbJnbixf5fCMCSCMhcH7N4l7u8Lj91XO+/QQTed83Xt7ARfP/v2wXH7dGi/vSr/+YNv5vzYvz/7cO3WpaN6XXx00n13XHpMzucFgFzc9lHmbsp0wpRcOcrPFQPC5vrXM3cBJqpr8BY4zyyv9jTYf+VG7yluNtU16n+e/cr1vhdGLUmblDdRNCddyvtL4GUnKHPR4+t7xf0d23x69pFd9Y8rvuEaIA3/69l6/8bv6CfdD2zZ5hZcpXLLeUfobz2P1rS7L5Qk3XXZsXH3/+aMw/JazBUAsvXWeO9fiqXih8+4BwGtwZA5lQU795XPj/OUbubXr2U32zbd2rPZzIpttjYp8Ho75vq9oc+kgqXq8AtBmYtdOndsuX1l94Pi7jPG6OrTDtGOnZK7K7vtvZO6d9tTHdq305IHL9Eb131bvz3zME+P+feeR+vPFxwZt+0Xpx2iw/beKW7bpd/cX5LU94bTWrZ98LvveHoMAEDqNArIzEvL1doa/wbXV2ZxrqPuGNAycSAqdlbtkDkVuublCb6VrRAIylz89NTtXZNnHdU1p3MYY3T6EXvHtbK9eE13vXBNd9f9f3d28hiyTh3aaehfz47b9q8fHa+Pfv9dnXbYXlra+1It7X2pjtpv17h9chnoCQBAUD7xKWfgu2UrM+8UYgRlLmK7CP1cGPebB+6mkw/ZQ5J07tH7tGzPlFbj8StP1O2XRLpLu3Rqr5MO3iPufrcezVF/Oydjebp0zH/pqONdkg4Cfljy4CVBFwEAioqgLAM/xoPOvu8ijel1rvbdtbP23KmTJt1xvp6/+mRJ0i9OO1jv3pi++/GKkw7Q9Wm6QXfs1EHfPTwyDm7C7edJkg7ac8ek/a4/41AduEdkEsNuXTpq4h3n68h9d85Y/un3XKgTDtpdUnIr3Kd/OD1ty9wp3fZI2raLM1niiH121tWnHSJp++SKsbeeG7fv0t6Xap9dwrkMzqEJXct+GnvrudqpU3v1uvhozb//4oI9TjH1/mF2E2ASp8IDQGtHnrIM/FizbMdOHeLGoEXX2pv7j57q1N6fuPit609L2vb81Sfrhj6Rqc5/63mUfn/213XZ8V/T958eo6tOOUg779BBg/58lkbMX6tfvjxB3z/xa9qtS0ede/Q+mrO6Rv8aMFevXnuKdu3cUe/99jstU9uP2GdnLajcnDRL1c17N35X1lodeusXLdum331h3Bfufd8/TlJkHccdOiS33g3685nasKVeh3XdWZOWbdCSdbX663vTsqugBEt7X6qXRy9R92576MPJ5Xr1q6WSpId+dLyWbdiip4ct0rz7e6qp2erYuwa6nuORn5ygbx28R86Zui8/4Wt68Iff1EdTyvXJlHKVLduoHTu11+z7ekqSZjn/S9Knf+ih/XfropvfnqxxizekOqUkafxt5+mRQfP0btlKHbHPzvrd2Yfrf9+N1Ne7v/2OunRsr8ufck+jsWvnDtpUtz0b/IXH7qs/nX+kpq6oynsW3lWnHqxeH3o7x6x7L5Ikzbz3In0jJhHq1acdotXVdRoyJ9xZ4nfo0C7nxbgBtF0mDHk7jDE9JT0hqb2kF621vdPt3717d1tWlv20X6/Wbd6m7vcPkSR9fFMPnei0EpWirfVNemHUYv3u7MPV0QkAxy1er5MP2aPl72xZa/XU0IW66tSD1dVpxVpbs02/eHG8+t5wmhqam/W/70zT41edqL2dAPTFUYt1f79IFuhMY97qGpp08ROjdN/3j9MZR7iP6Zu+skpD5lTqzCP21o+fHRt338++fbB+d9bhWr6hVp9NW6W7Lj9Wjw2erxdGLWnZJ7EMTc1W1VsbtOdOnZIeyy3omn3fRS2B9oCZa3Sjk+dnwQMXq9/01erYvp3OO2YfnfPv4VpdXafdunTUa78+Vde/Xqbjvrarnvn5yXFrsm1rbFJF9TYdvFdyC2es6q0NOuHeQZKkt68/TT99YVzSPtHnNmtVtY7df1cZY1TvBAidOkRe85venKxFazfr8atOlJHRRY+PlLS9yzAaRMfW0y19pySN+xj217O1e5eO2lLfqH7TV2vHHTrosm/ur0cHz1efccskSTvv0EEH7N5FA/98pmasrNblT43WtT266e7Lj9OtH86IGzj8wjXddf4x+8QF7UvWbdGWbY067mu7tmxvbrZ6c8JyXfyN/Vreq7GO2X9XWWt141mH60/vTJUUyTf4W+dHSs/j9tNR++2iJ75ckLKuR/3tHC3fUKufvzi+pW421TW21H86S3tfqopNdS0LIXsxpte56tF7aMvfe+3UqWXdxp7H7acBs1LPettnlx305wuO1PMjF+e8CkCiVIHlXy880nXB8DsvO1aTlm3QFzO8LQYfdmce2VV779SpZaD4fd8/Tnd9krxIN0pfMcZhG2MmWWvdB5XH7hd0UGaMaS9pvqQLJK2UNFHST621s1MdU+igTNr+Rcyg+fAbOGuNfttnUlyg5Cb6mn7r4N314e97+FqGjVvqtUPHdmkf3y/VWxs0bvF6XXTcfrLWavLyjTrhwN318KB5uvz4r+kbB2Q/zm/K8o06ar9dWsrf2NSsxmarzgnjDmMD1Im3n98SlLvZvK1RM1ZW6zuH75VyHykS5D/Qb4527dJRfzzviKzL/se3p2hjbb2+c/he+v3ZX3fdp7nZql07o3lrarSqaqvOccZ0btxSr5OclThOPGh3PXbliWpqbtbX99ml5diFlTWqqm1Qd2fs51/fm6b3J20fTHzo3jtpybotmnTH+Ro0u0JH7rtLy9jRaH29eu0p+ucXczQ/Jvt43xtO060fztCSdVt0/jH76sVfdpe1VqMXrtMD/ebo85tPV31Tsx4fskD/e8GRqt7aoIcGzNOEpeu1YkMkD9TR++2iuWtqtOTBS1oC1pdGL9HEJRt0bY9u+vZhe2lmebUeHzI/LlXCj08+UEftu4uO2X9XjVqwVs+N3J6hfdTfztFuO3bUrp07atHazTrvkREt9+28QwfNuOdCDZ1bqete2/4Z3P+WM1oC4SkrqnTA7l1cA9Ih/3uWzn90RNL2fXfdQT879RAd+7Vddf3rZTr7qK76v4uO0qX/8ZYY+ZrvHKIrTjpAvT6YHlfH8+7vqaPuyLz+YeeO7fSb0w/TXy48Uofe+oW+feieeue3kWElFzw6Qrt07qAPf9/D9Qfa0t6Xqq6hSUffuf1xRv7fOTp4rx2T9v/4ph664ukxkqQv/3KWzntkhM4/Zt+sW34H//lMXfDYyKyOKQRj0uf+2qVzB9XUNabewQc/+taBSetfZougLLYAxnxH0j3W2oucv2+VJGvtg6mOKUZQhtanvrFZzTY50IB39Y3NuvvTWXrgim+oXRvOmbdhS706ewjC12/epvFLNugSJ5VNbX2j5q2p0bFf27Wlq76+sbmlBbOQlq7bov126+x6/Q+dW6Ffv1rm+uW0eO1mnesEZtl+ef2p7xR9PHWVJt5+vrp0aq+dneXqttZHsrR36dRem7c1tmxP9MnUcu2xYyeddPDu2qVzRw2ZXaHu3fZQh/bttPMOHWSdvFTprsXJyzdqVdVWXXb811q2zVm9SYd33VnLN9TqsL13iju+tr5RHdu3c+1J2FrfpEufHKX/XHWS5x8/Yxau0+ND5uvt60+LSzIeq7nZqr6pWb9/c7J6fmM//e396ZHnf1MPHb3/Lmpulm5+e7KGzKlsCfiilq3fovKqrbrjo5n66KYe2nmHDnpp9GJddNx+OmSvnbS2ZptOeWCIPvz9d/XD/0bys91/xTe0106d1H/mGn06bVVL8HTwnjtq+YZa9bnuVO2xY6eUq4V8+oceOv5A9x6k6toGXffaRJUt26gFD1ysju3baWZ5tV4ctVgP/fgEzSiv0p477aBz/j1ckvTYlSfoz+/ED0e55bwj9OcLjlSfcct058czW7Yfte8umldR0/J39Hr8aMrKpHMkmn//xXpk0LyWHyA3nHmYbrukeAnZSyko+7Gkntba3zh/Xy3p29baPyTsd4OkGyTp4IMPPnnZsmVFLysAACieuoYmbdraoH127Swp0rKeavxxVHTNTGuttjU2a4cO7QKfOOQ1KCuZ2ZfW2uettd2ttd27ds0tdxgAACgdnTu2bwnIpMis7HQBWXSf6P+dO7YPPCDLRhiCsnJJsWnzD3S2AQAAtBlhCMomSjrCGHOoMaaTpKskfRpwmQAAAIoq8Dxl1tpGY8wfJA1UJCXGy9Za5h0DAIA2JfCgTJKstV9I+iLjjgAAAK1UGLovAQAA2jyCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAgBgjIAAIAQICgDAAAIAWOtDboMWTPGrJW0rMAPs7ekdQV+jFJDnSSjTuJRH8mok2TUSTLqJF5rq49DrLVdM+1UkmRxnIEAAAr3SURBVEFZMRhjyqy13YMuR5hQJ8mok3jURzLqJBl1kow6iddW64PuSwAAgBAgKAMAAAgBgrLUng+6ACFEnSSjTuJRH8mok2TUSTLqJF6brA/GlAEAAIQALWUAAAAhQFCWwBjT0xgzzxiz0BjTK+jyFJoxZqkxZoYxZqoxpszZtqcxZrAxZoHz/x7OdmOM+Y9TN9ONMd+KOc8vnf0XGGN+GdTzyYUx5mVjTKUxZmbMNt/qwBhzslPHC51jTXGfYfZS1Mk9xphy51qZaoy5JOa+W53nN88Yc1HMdtf3kzHmUGPMeGf7O8aYTsV7dtkzxhxkjBlmjJltjJlljLnF2d5mr5M0ddKWr5POxpgJxphpTp3c62x3fR7GmB2cvxc693eLOVdWdRVGaerjVWPMkphr5ERne6t/32RkreWf809Se0mLJB0mqZOkaZKODbpcBX7OSyXtnbDtIUm9nNu9JP3LuX2JpP6SjKTTJI13tu8pabHz/x7O7T2Cfm5Z1MGZkr4laWYh6kDSBGdf4xx7cdDPOcc6uUfSX132PdZ5r+wg6VDnPdQ+3ftJ0ruSrnJuPyvpd0E/5wz1sb+kbzm3d5E033nebfY6SVMnbfk6MZJ2dm53lDTeeU1dn4ek30t61rl9laR3cq2rMP5LUx+vSvqxy/6t/n2T6R8tZfFOlbTQWrvYWlsvqa+k7wdcpiB8X9Jrzu3XJF0Rs/11GzFO0u7GmP0lXSRpsLV2g7V2o6TBknoWu9C5staOlLQhYbMvdeDct6u1dpyNfIK8HnOu0EpRJ6l8X1Jfa+02a+0SSQsVeS+5vp+cX7LnSnrfOT62fkPJWrvaWjvZuV0jaY6kA9SGr5M0dZJKW7hOrLV2s/NnR+efVernEXv9vC/pPOd5Z1VXBX5aOUtTH6m0+vdNJgRl8Q6QtCLm75VK/yHTGlhJg4wxk4wxNzjb9rXWrnZur5G0r3M7Vf20xnrzqw4OcG4nbi9Vf3C6FV6OdtUp+zrZS1KVtbYxYXtJcLqYTlLkVz/XiZLqRGrD14kxpr0xZqqkSkWCh0VK/Txanrtzf7Uiz7vVfNYm1oe1NnqNPOBcI48ZY3ZwtrWp940bgjKcbq39lqSLJd1kjDkz9k7n10ebnqJLHbR4RtLhkk6UtFrSI8EWp/iMMTtL+kDSn6y1m2Lva6vXiUudtOnrxFrbZK09UdKBirRsHR1wkQKVWB/GmG9IulWRejlFkS7JvwdYxFAhKItXLumgmL8PdLa1Wtbacuf/SkkfKfIhUuE0C8v5v9LZPVX9tMZ686sOyp3bidtLjrW2wvmAbZb0giLXipR9naxXpFuiQ8L2UDPGdFQk+HjTWvuhs7lNXyduddLWr5Moa22VpGGSvqPUz6PluTv376bI8251n7Ux9dHT6fq21tptkl5R7tdISb5v0iEoizdR0hHOTJlOigy8/DTgMhWMMWYnY8wu0duSLpQ0U5HnHJ3d8ktJnzi3P5V0jTND5jRJ1U7XzUBJFxpj9nC6Ki50tpUyX+rAuW+TMeY0Z6zINTHnKinR4MPxA0WuFSlSJ1c5M8kOlXSEIoNvXd9PTovSMEk/do6Prd9Qcl67lyTNsdY+GnNXm71OUtVJG79Ouhpjdndud5F0gSJj7VI9j9jr58eShjrPO6u6Kvwzy02K+pgb80PGKDIGLPYaadXvm4zcRv+35X+KzP6Yr8g4gNuDLk+Bn+thiszemSZpVvT5KjKm4UtJCyQNkbSns91IetqpmxmSusec69eKDEZdKOnaoJ9blvXwtiLdLA2KjEm4zs86kNRdkQ+dRZKekpO0Ocz/UtRJH+c5T1fkw3P/mP1vd57fPMXMfkr1fnKuvQlOXb0naYegn3OG+jhdka7J6ZKmOv8uacvXSZo6acvXyfGSpjjPfaaku9I9D0mdnb8XOvcflmtdhfFfmvoY6lwjMyW9oe0zNFv9+ybTPzL6AwAAhADdlwAAACFAUAYAABACBGUAAAAhQFAGAAAQAgRlAAAAIUBQBqDojDFLjTHWw7+zgy6rF8aYe5zy3hN0WQCUrg6ZdwGAghmoyJqRqaS7DwBaFYIyAEHqba0dHnQhACAM6L4EAAAIAYIyAKFnjOnmjNlaaozpYIzpZYyZY4ypM8ZUGGNeM8YcnOb444wxrxtjVhhjthlj1hljvjDGXJzhcS8yxnxojFlljKk3xqwxxowxxvzdWcvP7Zh9jTHPGWNWOo+1xBjT2xjT2WXf9saYG40xXxljqp3HqDDGTDbGPGKM6Zp9bQEoVQRlAErNO5LulbRc0seStimyEPFEY8xRiTsbY74naZKkqyVVS/pA0mxJF0n6whjzD5djjDHmGUkDFFlUu9w5bpqkgyT1lrSvS9kOch7rMkljJQ2XtI+kv0t612X/lyQ9I+lESeMlve88xm6S/lfS4RnqAkArwpgyAKXkEEldJJ1krZ0tScaYTooEN79QZDHsU6M7G2P2c7btIOkv1tpHY+47W1I/SXcYY0ZbawfGPM4tkm6UVCHpCmvtuJjjjKRzJG10Kd+vJb0o6SZrbb2z/zGKLDZ9uTGmh7V2jLP9EEm/lLRC0inW2orYExljTpS0KqvaAVDSaCkDEKRhadJhVKU45h/RgEySnODnZkmbJJ1ijOkRs+/1knaVNCY2IHOOGy7pSefPv0a3G2M6SLrd+fNXsQGZc5y11g611la7lG2FpD9GAzJn/zmKBIaSdF7Mvvs4/09ODMic46ZaaytdHgNAK0VLGYAgpUuJUZti+xuJG6y1VcaYzyT9XNLZksY4d53l/P9ainO9rEjX4unGmPbW2iZJ3SXtLWmltXZAxmcQb6i1dqvL9rnO/19L2FYj6VJjzG2S3rTWLsvy8QC0IgRlAIKUbUqMKmttqha0pc7/B8ZsO8D5f0maY5oldZa0l6RKRbpIJWleFuWKWp5i+ybn/5bB/tbaGmPMrxUJDB+Q9IAxplyRsWj9JPW11tblUAYAJYruSwBtgS3Qvomas9nZWvu+pIMl/UqR4GyzpB9LekXSXGPMQXmUBUCJISgDUEp2N8bsluK+bs7/5THborcPS3NMO0l1kjY426KtXUkzOQvBWltlrX3NWnudtfZoSV+XNEyRFrt/FaMMAMKBoAxAqfl54gYnULvM+XN4zF0jnP+vSXGua53/R1trG53bkyStk3SgMeai/IqaPWvtIkW6MyXphGI/PoDgEJQBKDV3OWkmJEnGmI6SnlAkt9cka+3omH1fUGQw/enGmD/GnsQYc6YiszYl6ZHodmttg6QHnT9fMcacmnCcMcack6bFzhNjzEnGmCtTJKG93Pmfgf9AG8JAfwBB6mWM+VWa+9+y1g6K+Xu5Ii1ZU40xQxVJBvtdRZK2rlNCi5i1do0x5mpFEs4+YYz5jaSZisyCPEORH6b3u8yyfEzSMZJ+I2mcMaZM0kJJe0o61nm8Q53Hz9UhkvpKqjXGTFYknUYnSScp0t1aI+muPM4PoMQQlAEIUqbuwamSYoMyK+knknopkqH/EEVmNr4h6U5r7dLEE1hrPzHGdFck9cW5igykr3HO+6S19guXY6yk640xnyiSRPZURbLur1ckOHtSqVN5eDVO0q2KpO04WtLJkuoVCc4eccpGSxnQhpjIZw8AhJcxppsiaS2WWWu7BVoYACgQxpQBAACEAEEZAABACBCUAQAAhABjygAAAEKAljIAAIAQICgDAAAIAYIyAACAECAoAwAACAGCMgAAgBAgKAMAAAiB/wctBjTYc5pvIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.plot(losses)\n",
    "plt.xlabel(\"Epochs\",fontsize=22)\n",
    "plt.ylabel(\"Loss\",fontsize=22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Games played: 1000, # of wins: 927\n",
      "Win percentage: 92.7%\n"
     ]
    }
   ],
   "source": [
    "max_games = 1000\n",
    "wins = 0\n",
    "for i in range(max_games):\n",
    "    win = test_model(model, mode='random', display=False)\n",
    "    if win:\n",
    "        wins += 1\n",
    "win_perc = float(wins) / float(max_games)\n",
    "print(\"Games played: {0}, # of wins: {1}\".format(max_games,wins))\n",
    "print(\"Win percentage: {}%\".format(100.0*win_perc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial State:\n",
      "[['P' ' ' ' ' '-']\n",
      " [' ' 'W' ' ' ' ']\n",
      " [' ' ' ' '+' ' ']\n",
      " [' ' ' ' ' ' ' ']]\n",
      "Move #: 0; Taking action: r\n",
      "[[' ' 'P' ' ' '-']\n",
      " [' ' 'W' ' ' ' ']\n",
      " [' ' ' ' '+' ' ']\n",
      " [' ' ' ' ' ' ' ']]\n",
      "Move #: 1; Taking action: r\n",
      "[[' ' ' ' 'P' '-']\n",
      " [' ' 'W' ' ' ' ']\n",
      " [' ' ' ' '+' ' ']\n",
      " [' ' ' ' ' ' ' ']]\n",
      "Move #: 2; Taking action: d\n",
      "[[' ' ' ' ' ' '-']\n",
      " [' ' 'W' 'P' ' ']\n",
      " [' ' ' ' '+' ' ']\n",
      " [' ' ' ' ' ' ' ']]\n",
      "Move #: 3; Taking action: d\n",
      "[[' ' ' ' ' ' '-']\n",
      " [' ' 'W' ' ' ' ']\n",
      " [' ' ' ' '+' ' ']\n",
      " [' ' ' ' ' ' ' ']]\n",
      "Game won! Reward: 10\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_model(model, mode='random')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Listing 3.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "l1 = 64\n",
    "l2 = 150\n",
    "l3 = 100\n",
    "l4 = 4\n",
    "\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(l1, l2),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l2, l3),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(l3,l4)\n",
    ")\n",
    "\n",
    "model2 = copy.deepcopy(model) #A\n",
    "model2.load_state_dict(model.state_dict()) #B\n",
    "\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "learning_rate = 1e-3\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "gamma = 0.9\n",
    "epsilon = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2999 0.02813359536230564\n"
     ]
    }
   ],
   "source": [
    "from collections import deque\n",
    "epochs = 5000\n",
    "losses = []\n",
    "mem_size = 1000\n",
    "batch_size = 200\n",
    "replay = deque(maxlen=mem_size)\n",
    "max_moves = 50\n",
    "h = 0\n",
    "sync_freq = 500 #A\n",
    "j=0\n",
    "for i in range(epochs):\n",
    "    game = Gridworld(size=4, mode='random')\n",
    "    state1_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/100.0\n",
    "    state1 = torch.from_numpy(state1_).float()\n",
    "    status = 1\n",
    "    mov = 0\n",
    "    while(status == 1): \n",
    "        j+=1\n",
    "        mov += 1\n",
    "        qval = model(state1)\n",
    "        qval_ = qval.data.numpy()\n",
    "        if (random.random() < epsilon):\n",
    "            action_ = np.random.randint(0,4)\n",
    "        else:\n",
    "            action_ = np.argmax(qval_)\n",
    "        \n",
    "        action = action_set[action_]\n",
    "        game.makeMove(action)\n",
    "        state2_ = game.board.render_np().reshape(1,64) + np.random.rand(1,64)/100.0\n",
    "        state2 = torch.from_numpy(state2_).float()\n",
    "        reward = game.reward()\n",
    "        done = True if reward > 0 else False\n",
    "        exp =  (state1, action_, reward, state2, done)\n",
    "        replay.append(exp) #H\n",
    "        state1 = state2\n",
    "        \n",
    "        if len(replay) > batch_size:\n",
    "            minibatch = random.sample(replay, batch_size)\n",
    "            state1_batch = torch.cat([s1 for (s1,a,r,s2,d) in minibatch])\n",
    "            action_batch = torch.Tensor([a for (s1,a,r,s2,d) in minibatch])\n",
    "            reward_batch = torch.Tensor([r for (s1,a,r,s2,d) in minibatch])\n",
    "            state2_batch = torch.cat([s2 for (s1,a,r,s2,d) in minibatch])\n",
    "            done_batch = torch.Tensor([d for (s1,a,r,s2,d) in minibatch])\n",
    "            Q1 = model(state1_batch) \n",
    "            with torch.no_grad():\n",
    "                Q2 = model2(state2_batch) #B\n",
    "            \n",
    "            Y = reward_batch + gamma * ((1-done_batch) * torch.max(Q2,dim=1)[0])\n",
    "            X = Q1.gather(dim=1,index=action_batch.long().unsqueeze(dim=1)).squeeze()\n",
    "            loss = loss_fn(X, Y.detach())\n",
    "            print(i, loss.item())\n",
    "            clear_output(wait=True)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            losses.append(loss.item())\n",
    "            optimizer.step()\n",
    "            \n",
    "            if j % sync_freq == 0: #C\n",
    "                model2.load_state_dict(model.state_dict())\n",
    "        if reward != -1 or mov > max_moves:\n",
    "            status = 0\n",
    "            mov = 0\n",
    "        \n",
    "losses = np.array(losses)\n",
    "\n",
    "#A Set the update frequency for synchronizing the target model parameters to the main DQN\n",
    "#B Use the target network to get the maiximum Q-value for the next state\n",
    "#C Copy the main model parameters to the target network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Loss')"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAm4AAAG3CAYAAAANcdlpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzs3XecXGW9x/Hvb5PQSxRCB0NTiggoTQEFRUBA8Cp6QS+CXnu/ctUACparggUQUIqAFJFeJUAoCSmkJ5Dee91s2maTTbLtuX/Mmd0zs2dmzsw5U87u580r7O7MmTPPnDnld57ye8w5JwAAANS+umoXAAAAAOEQuAEAACQEgRsAAEBCELgBAAAkBIEbAABAQhC4AQAAJASBGwAAQEIQuAEAACQEgRsAAEBC9K12Acpl7733dgMHDqx2MQAAAAqaNGnSWufcgELL9djAbeDAgZo4cWK1iwEAAFCQmS0JsxxNpQAAAAlB4AYAAJAQBG4AAAAJQeAGAACQEARuAAAACUHgBgAAkBAEbgAAAAlB4AYAAJAQBG4AAAAJQeAGAACQEFUP3MzsYDMbZmYzzWyGmf0wYJmzzKzRzN7x/l1fjbICAABUUy3MVdom6Wrn3GQz213SJDN7zTk3M2u5kc65i6pQPgAAgJpQ9Ro359wq59xk7/cmSbMkHVjdUgEAANSeqgdufmY2UNKJksYFPP1hM5tiZi+b2bE5Xv8NM5toZhMbGhrKWFIAAIDKq5nAzcx2k/S0pB855zZlPT1Z0nucc8dLul3Sc0HrcM7d45w7yTl30oABA8pa3m2t7Zq9epM2bWst6/sAAACk1UTgZmb9lAraHnHOPZP9vHNuk3Nus/f7S5L6mdneFS5mhgUNm3X+rSM1ZsG6ahYDAAD0IlUP3MzMJN0naZZz7uYcy+znLSczO0WpchMxAQCAXqUWRpWeLukKSdPM7B3vsWslHSJJzrm7JF0q6dtm1iZpq6TLnHOuGoUFAAColqoHbs65UZKswDJ3SLqjMiUCAACoTVVvKk066v0AAEClELiVyPJXEgIAAMSOwA0AACAhCNwAAAASgsANAAAgIQjcImN0AgAAqAwCtxIZYxMAAECFEbgBAAAkBIEbAABAQhC4AQAAJASBW0TMnAAAACqFwK1EDE4AAACVRuAGAACQEARuAAAACUHgBgAAkBAEbhExNgEAAFQKgVuJTIxOAAAAlUXgBgAAkBAEbgAAAAlB4AYAAJAQBG4RMXMCAACoFAK3EjFzAgAAqDQCNwAAgIQgcAMAAEgIAjcAAICEIHCLyDF3AgAAqBACtxIxNgEAAFQagRsAAEBCELgBAAAkBIEbAABAQhC4RcTMCQAAoFII3ErEzAkAAKDSCNwAAAASgsANAAAgIQjcAAAAEoLALSLGJgAAgEohcCsZoxMAAEBlEbgBAAAkBIEbAABAQhC4AQAAJASBW0SOqRMAAECFELiViJkTAABApRG4AQAAJASBGwAAQEIQuAEAACQEgRsAAEBCELiViLEJAACg0gjcAAAAEoLADQAAICEI3AAAABKCwC0iJk4AAACVQuBWImPqBAAAUGEEbgAAAAlB4AYAAJAQBG4AAAAJQeAGAACQEARuETkxrBQAAFQGgVuJGFMKAAAqjcANAAAgIQjcAAAAEqLqgZuZHWxmw8xsppnNMLMfBixjZnabmc03s6lm9sFqlBUAAKCa+la7AJLaJF3tnJtsZrtLmmRmrznnZvqW+ZSkI71/p0q60/tZdUx5BQAAKqXqNW7OuVXOucne702SZkk6MGuxSyQ95FLGSupvZvtXuKgZmPEKAABUWtUDNz8zGyjpREnjsp46UNIy39/L1T24AwAA6NFqJnAzs90kPS3pR865TSWu4xtmNtHMJjY0NMRbQAAAgCqricDNzPopFbQ94px7JmCRFZIO9v19kPdYBufcPc65k5xzJw0YMKA8hQUAAKiSqgduZmaS7pM0yzl3c47FXpD0ZW906WmSGp1zqypWyDwYnAAAACqlFkaVni7pCknTzOwd77FrJR0iSc65uyS9JOkCSfMlNUv6ShXKmcGYOwEAAFRY1QM359woFZhByjnnJH23MiUCAACoTVVvKgUAAEA4BG4AAAAJQeAWEWMTAABApRC4lYiZEwAAQKURuAEAACQEgRsAAEBCELgBAAAkBIFbRI6pEwAAQIUQuAEAACQEgRsAAEBCELgBAAAkBIEbAABAQhC4RcTQBAAAUCkEbiVi5gQAAFBpBG4AAAAJQeAGAACQEARuAAAACUHgFhWjEwAAQIUQuJXIGJ0AAAAqjMANAAAgIQjcAAAAEoLADQAAICEI3CJyjE4AAAAVQuBWIoYmAACASiNwAwAASAgCNwAAgIQgcAMAAEgIAreIHGMTAABAhRC4lYiJEwAAQKURuAEAACQEgRsAAEBCELgBAAAkBIFbRIxNAAAAlULgViJj7gQAAFBhBG4AAAAJQeAGAACQEARuAAAACUHgFhEzJwAAgEohcCsRMycAAIBKI3ADAABICAI3AACAhCBwAwAASAgCt4gccycAAIAKIXArEWMTAABApRG4AQAAJASBGwAAQEIQuAEAACQEgVtEzJwAAAAqhcCtVIxOAAAAFUbgBgAAkBAEbgAAAAlB4AYAAJAQBG4RMTYBAABUCoFbiYzRCQAAoMII3AAAABKCwA0AACAhCNwAAAASgsAtKqZOAAAAFULgViJjbAIAAKgwAjcAAICEIHADAABIiKoHbmZ2v5mtMbPpOZ4/y8wazewd79/1lS4jAABALehb7QJIekDSHZIeyrPMSOfcRZUpTnEYmgAAACql6jVuzrkRktZXuxzFYmwCAACotKoHbiF92MymmNnLZnZstQsDAABQDbXQVFrIZEnvcc5tNrMLJD0n6cigBc3sG5K+IUmHHHJI5UoIAABQATVf4+ac2+Sc2+z9/pKkfma2d45l73HOneScO2nAgAEVLScAAEC51XzgZmb7maXS3ZrZKUqVeV11S9WFiRMAAEClVL2p1MwelXSWpL3NbLmkGyT1kyTn3F2SLpX0bTNrk7RV0mXOVT9cMqZOAAAAFVb1wM05d3mB5+9QKl0IAABAr1bzTaUAAABIIXADAABICAK3iGqgux0AAOglCNxKxNAEAABQaQRuAAAACUHgBgAAkBAEbgAAAAlB4BYRQxMAAEClELiViIkTAABApRG4AQAAJASBGwAAQELENlepmdVJ+qqk4yUtkXS3c64prvUDAAD0dkXXuJnZIDNrNrOzsp4aLOluSd+VdJOkMWa2a/Qi1jYmTgAAAJVSSlPpeZI2SRqefsDMzvUeXyHp/ySNl3S0UjVwPZIxdwIAAKiwUgK3IyTNdJmTdH5OqcwYlznnrpf0cUkbJH0xehEBAAAglRa47S1pVdZjZ0ha7ZwbLUnOua2SRksaGKl0AAAA6FRK4NYhqbPvmpntKekoSW9lLdcoqX/pRQMAAIBfKYHbIkmneqNIJekiSSZpVNZyAyStjVC2RGBsAgAAqJRSArcXJO0r6Vkz+4GkP0pql/R8egEzM0knKhXk9UyMTQAAABVWSuB2k6RZkj4t6VZJ+0n6k3NuiW+ZM5SqccuuhQMAAECJik7A65xrNLOTJF2qVM3bBOfc8KzF9pL0F0mPRS8iAAAApBJnTvBGjT6c5/nnJD1XaqEAAADQXexzlZrZXmbWJ+711irH1AkAAKBCSpny6gQz+6mZHZX1+LlmtkzSGkkNZvb1uApZi4zBCQAAoMJKqXH7vqTfKTXtlSTJzPaV9IykA5XKkNFf0p1mdnIchQQAAEBpgdtHJE11zq30PfZlSbsoNcp0J0mf9db9/cglBAAAgKTSArd9JC3LeuwcSa2SfuWca/MGJ0yUdGrE8gEAAMBTSuC2u6TNWY+dImmyc67R99gCpZpOAQAAEINSArcNkt6T/sPMTpC0p7rPVVqnVC1cj8TYBAAAUGmlBG4TlZqrNN0M+j9KDUgYmrXckZJWRSgbAAAAfEoJ3P4iqY+k0Wa2TtIVkhZKGpJewMz2lnScpHfiKCQAAABKCNycc69K+qqkJZJ2lPSmpE8759p9i12hVHD3ZvQiAgAAQCp9yqsHJD2QZ5G7JN2v7oMYehwmTgAAAJVSUuBWiDeX6dZyrLtWGFMnAACACosUuJnZgZI+qq60HyskjXDOrYhaMAAAAGQqKXAzs/6S/irpC+reT67DzB6X9D3n3MaI5QMAAICn6MDNzHZWKvXH8UqlARmr1KhSSTpMqdkSLpd0tJmd4TWbAgAAIKJSatx+JOkESaMlfd05N8v/pJkdLeluSadL+oGkm6IWspY5MToBAABURil53L6g1OwJF2YHbZLkPXaxpI2SLotWvNrF0AQAAFBppQRuR0oaljUvaQavb9swb1kAAADEoJTADQAAAFVQSuA2X9JZZrZ7rgXMbA9JZ3nLAgAAIAalBG5PSnq3pBfM7IjsJ73HnpX0LklPRCte7WPmBAAAUCmljCq9RdJ/SvqYpFlmNlbSIqVSgxwm6TSl5imdJunWmMpZc5g4AQAAVFrRgZtzrtnMzpZ0p6TPKZX243T/IpKekvRt51xzLKUEAABAyZPMr5P0BTM7RNKZypzyaqRzbqmZ7WtmhzjnlsZUVgAAgF4t0lylXlD2SI6nn5N0ctT3AAAAQEq504H0+J5gjE0AAACVQh63ElnPj0kBAECNIXADAABICAI3AACAhCBwAwAASAgCt4iYOQEAAFRKwVQdZvbREte9R4mvSwRmTgAAAJUWJsfamyot64WV+DoAAAAECBO4LRUBGAAAQNUVDNyccwMrUA4AAAAUwOCEiByVkQAAoEII3AAAABKCwA0AACAhCNwAAAASouqBm5ndb2ZrzGx6jufNzG4zs/lmNtXMPljpMgIAANSCqgdukh6QdH6e5z8l6Ujv3zck3VmBMoXGzAkAAKBSqh64OedGSFqfZ5FLJD3kUsZK6m9m+1emdLkxcwIAAKi0qgduIRwoaZnv7+XeY92Y2TfMbKKZTWxoaKhI4QAAAColCYFbaM65e5xzJznnThowYEC1iwMAABCrJARuKyQd7Pv7IO8xAACAXiUJgdsLkr7sjS49TVKjc25VtQsFAABQaWEmmS8rM3tU0lmS9jaz5ZJukNRPkpxzd0l6SdIFkuZLapb0leqUNJOJ0QkAAKCyqh64OecuL/C8k/TdChUHAACgZiWhqRQAAAAicAMAAEgMAreIHFMnAACACiFwKxEzJwAAgEojcAMAAEgIAjcAAICEIHADAABICAK3iBibAAAAKoXArUSMTQAAAJVG4AYAAJAQBG4AAAAJQeAGAACQEARuETE2AQAAVAqBW4mMqRMAAECFEbgBAAAkBIEbAABAQhC4AQAAJASBW0TMnAAAACqFwK1EDE0AAACVRuAGAACQEARuAAAACUHgBgAAkBAEbhE55k4AAAAVQuBWIiZOAAAAlUbgBgAAkBAEbgAAAAlB4AYgMTZvb9OW7W3VLgYAVE3fahcg6Zg5Aaic998wRJK0+MYLq1wSAKgOatxKZIxOAAAAFUbgBgAAkBAEbgAAAAlB4AYAAJAQBG4RMTYBAABUCoEbAABAQhC4AQAAJASBGwAAQEIQuAEAACQEgVtUTJ0AAAAqhMAtAiZPAAAAlUTgBgAAkBAEbgAAAAlB4AYAAJAQBG4RMTQBAABUCoFbBIxN6Dla2zv048ff0bL1zdUuCgAAORG4AZLGLlynZ95eoWuemVbtoiAhnpm8XHcMnVftYgDoZfpWuwAAkEQ/fmKKJOl7Hz+yyiUB0JtQ4wYAAJAQBG4RMXECAACoFAK3CIypEwAAQAURuAEAACQEgRsAAEBCELgBMXptZr2ef2dFtYsBAOihSAcSkWPuBPh8/aGJkqRLTjiwyiUBAPRE1LhFwNAEAABQSQRugEjrAgBIBgI3AACAhCBwA3wmL91Q7SIAAJATgVtENLH1DJu3t0mSmlvaq1wShNHRwYEHoHcicIvATIwpBapg2Jw11S4CAFQFgVsEJqPGrYfge0yW1na+MAC9U00EbmZ2vpnNMbP5ZjYo4PmrzKzBzN7x/n2tGuXMlqpx4wICAAAqo+qBm5n1kfRXSZ+SdIyky83smIBFH3fOneD9u7eihczBjJoa1K4HRy/WBX8ZWe1iAABiVPXATdIpkuY75xY651okPSbpkiqXKZRUUymRW09gPTCb8g0vzNDMVZuqXQwAyGv+miYt39AcaR3L1jfr4TGLYylPrauFwO1ASct8fy/3Hsv2OTObamZPmdnBlSlaftS4RTNx8Xp96DevqXFra7WLAiAmCxo26/5Ri6pdDCTIOTeP0Bk3DYu0ji/dO06/eH5Gr7ie1ELgFsa/JQ10zn1A0muSHgxayMy+YWYTzWxiQ0ND2QtlYlRpFLe+Pk/rtrRoyrKN1S6KNjS3VLsIKEJPrCGtBXPrm9TS1hFpHV+4a4x+/eLMyOsBirHRO4f3hlawWgjcVkjy16Ad5D3WyTm3zjm33fvzXkkfClqRc+4e59xJzrmTBgwYUJbC+tUZo0qjqKWL73XPTq92EVAEjrv4rWrcqnNvGaFfvzgj0nrSuRBb2wncUDnmXVB6w7mhFgK3CZKONLNDzWwHSZdJesG/gJnt7/vzYkmzKli+3EzqCLGXzK1v0rDZ0fJObWtt18BBg3XvyIWR1lOLesFxBtS8jc2pJqaJi6PNHpK+IeO4RiX1pv2u6oGbc65N0vckDVEqIHvCOTfDzH5tZhd7i/3AzGaY2RRJP5B0VXVKmylshdG5t4zQVx6YEOm90ifVe0b0vMANmdZvadHAQYM1duG6ahcFvUj6whfmZjTvemIoC1Cs9H7XG5pK+1a7AJLknHtJ0ktZj13v+/0aSddUulyFmFVuVGktNSvGrTccaMWYvCRV43HPiIU67bC9qlwa9BameJuaOK5RSZ1NpVUuRyVUvcYtySo55VWUc+Dn7hytJycuK7xghcV1oA2f26DpKxqjF6hGdFb518iFb+Cgwbr5tbnVLgbKLK6mpt50AUXtSNdtnPR/r9fk9S5OBG4RJGVwwqQlG/STp6ZWuxjdxFWJeOX943XR7aNiWlv11WLt6m1vzKt2ETLU4jZKurqYbhi6mqyilQcohv+c8MsXog2wqXUEbhGYovcHCf1ePflCxQk+UE/aLC1tHVqxcWts6yMoKIeYmko7I7eI6wEQiMAtgko2lfZEXU0zbEW/uPsa1YJBT0/V6TcOVXNLW7WLEtn6LS2a1QNnpIh7VB7HNSqrJ9duZKqJwQnJVfmm0p50Kuw9h1mReuCw9qFzUulwtrV2aJcdqlyYiC68baRWNW6rdjFiF9eoPJpKUQ3+VqmevutR4xZBakdJ7SLNLW1qK2PCyZ4c5HCCz9QTv+ueNFS/JwZtUnyDChicgGqI67w5ftF6PTZ+aUxrKw8CtwjqfHOVHnP9EH33X5OrW6CE6U2ZrkvRE4KcNC7mtS+umrJaGxWN3iGufuBfuHuMBj0zLZ6VlQmBWwQmyxicMGRGfRVLkzw9sWYpDtYDR6LE3XxWS5uoo6NnBCh1ncF1TE2lEcuTdN/+56SMmW5a2jr04OjFau8h+0utsV50RSFwi8Cs8rVF3MT2fD2xj1AtDUTZtK1V33x4otZt3l544TLbsr1NV9w3TsvWN1e7KF0zJxTo8fHGrHotWrslz3pSK6rUiPta9fL01fq/wV2zM941fIFueGFGj88xVi21dDNXbgRuEZi4q4wD2zBTLQU58Yl3xEWUmOCx8Us1ZEa97hq+IJ7CRPDazHqNnLdWfxwyp9pFCe2/H5yos//0Zs7nO6+fPWn3jcGmralpC5u2JX9kdS3qRXEbgVsUVoUEvD3proK+MMFqKR1IXN9NLe23fetSp73W9upv4FqaGLuuLp6aslr6TLUkru2Lwnr6JiZwiyCVx62ye0iUHbKlrUOzVm3S9c9Pr5FgqYau5jWkK6CtbjnKIa6PNGX5xpjWFF2Uz1RLzYpxN9HXwEeqKZ1N0WyXsuiJfYNzIXCLoKJ93GLYJ1vaO3TFfeP00JglaqiB/j1pvfE8NmbBOl15//jAjspxn35qIUiPOyi4Z8TCwgvlKktWTe8r01dpbZWOh7rODVOVt88Q33UvnkEOPU1dDQXpSDYCtwhMVrmLYo0d6xMXr9fqiPmsenLNUiHf/ddkDZ/boI3NLTmXievCt621fPkFw4q7316UGMP/2k3bWvWtf07WVf8YH7VIJZal9i7mkUeVVvi4XtO0Tdta2yvzZhHENRcsgtX1omimF33U+FVnyqvaOOgvvWuMPvHnNyOto+sCWhufqWbEfOEr9kLc0LRdAwcN1lOTlseXviPmfntx7THtXj+35Rvim0e1GHEFOWs2bdPs1dGm4arV76iQU377hq68vzqBd9rL01bp/FtH5E0N0xWkV6pUvUsfmkoRRl0FByfE8Tbm+38ctrREu8vtRcdZUTovoDGtr9h9NJ3q4fEJ8WUPL0eH9db2Dg0cNFgPjVlcZFni3r6lr6kupprIM24apvNvHRlpHXH1warGLBnjFq2v2HsF+dHj72j26ia15Jk9p64XtzBUQp+63nNBIXCLwFS5Jo7Y36YGTh5dfT6qXJAyGzZ7jZauC5+nqycGtOXot7fZS6tw82tziytLwAW0WhdTi+kYyBcwFC9ZTaWV0NbeETlxbi0NROmJ6nwnzq0JaDqPgsAtiio0lcY1HU0t6LrD79knsq88MEFn/WlY6OXTF4iJi+OpRYiydWO/X6ix77rax0MtzOH68rRVGQmAI59jeuBo8SOue1kX3zGq4HL5th3pj8qr2sdyJRG4ReCbY77s4ujU7XL8Xi29aa7SYm7Wd9mhT9Gvyf/e1d/AcX/X/qH/pa6zFkY91sIx8O1HJuuCv3Q1s8a1v0RdzbDZa7RyY3X6HgaZsTJ3H8IwQUNdTE3021rbtTViNxUEa9rWWu0ihELgFoGZxXbyb25py3snln4qyl3FvPqmmroXZnh8sB36pg7Lo/bbPZb1Fbt5k1Aj4Jwr+ViIPV9ZhNfW1Uitc9P2rmz+UUsS1wjirzwwQeffOiJiaSor32dO73dRv+tTf/eGjr7+lUjrCOODv3lNX67yoI9i1MVQ5Xb538fGUJLyI3CLoC6mPG4bm1t0zPVDdOg1LxW8k4ryfv4+GrVwbe6Jc3LGIZ3ZP7aLeYTVxD1zQk181zXUplJL/TzT3030ptJ41iNJm3rQ9FBx7XaNW6PXCm3e3qaL7xilOaubci6zfkuLRsxtiPxeSTJ9RbSR2ZVC4BaByWK5uG5s7joQ73xzfuAycZzXnWrqmhXbiLqeJr1d5tZv1votufO8hVXsPlqODOS1OP9qRteBKkWUtTg9VNRtsdLL7+gkjZzXoIGDBkdOVdKT1MLNy5gF6zR1eaP+8MrsahclNnHUuCUFgVsEuWZOaO9wWtUYvm9GumlMkra3BY8Oi+PC4l/FP95aFHl9UXXWNuQYEBc1wa8kbdlem3fs6YBsQ3P+u+e59bnviMOqdjOc1JVjqRZqlvw1QnF0pI+yfTv7PdXAd5QWpSQLGjZ3rcc5vTJ9tSRpwuINEUtV28LsR3GnoYmij3fJaa+h/S6qXhS3EbhFFbTb/2HIbH3490NVvykz8Mh1cu7bp2uPKzTkPNIIQec6TzB3R5gyKJdl65v1xyGzQ1+E8g2PHzG3Qaf9/o3OE38ppi1v1LE3DNHL01aVvI5ye3tp/gtaHOfVYoOlcgQRcfdnjFJbFvcJfvT8dSW/Nl2WTTE0f8UmwlfkryHOWE0Z9qntbe1qLHDjU0tqqbtA+niMmuIkitEL1qq5Jb4ba2rcEIrlSMA7fE6qX8C6zZnNXLkOWP/dWq7jqBYO9kK+/cgk/XXYAs2t31x4YeU/kU1b0Sgp2mTiU1ekXjtiXnA/jflrmqp64qqUUpsnTRZb7UA5UiGUWlv28rT0zUA8ZWmNkEMtfbGZsrwxlrLEIY4aRMmr0SzjtfTK+8fr+F+/Wr43KEHedCCqndrVdLLaqDdSfxwyWwMHDS76dSs2btUX/z5O//vklEjv79eL4jYCtyhSgxPyjATNujDkOkj8y5WzWcvMyrpzt7alyh42UOiaX7vwaNpS5JvCZ259k865eYRue2Ne6W+QEGG2YVt7h258eXbG3Klx9kfrusOPZ33+3bjYUo6av7bbY5u2tamjw+mBtxZVdN7LWrzYRPnWM7PXlzdAGbuwurMl+IX5HtPLbN7epoGDBuvu4QvKW6g8+sRU4/bXYaV9hmavC8vc+s15a91a2jo0NeTNew0eSmVD4BZBnXWvkRi7cJ2ac4wMDXOI5AoE44jnauFOz6+rf0/35+K4oOWr0Vvl9Z+bXKCpslRRtnXc31KYm4EhM+p11/AF+vWLM8syOCGuO/w0/1pacvQLLbiOrKIMnrZKv/z3TP351TmlF6xItdi8E+Ur6pNV41Ztzjmt2RS9r2wc0lsm3Zz88NglVStLXfp4jHPCjSL4k6/nqzn/7eCZuviOt7SwIUQrTg0eS+VC4BZBnXW/Y7nsnrFauj54eqMwF61yt9xVatd+a/7avAM0Ji5er1dnppqsyvWZS/2scQS4cX2mWBIvF1hFc0tbZ3NfS1tHxuePO2Fu2Dv8xq2tofu/5BrQU0j2Z0u/XxzpFsKKO3ALu+/OX9OkobPrA5+L1FTqu6L419KwOfro6FLcPWKhTvndG1rszb9bTdk3ktUMbMO0dlSkBAWa06d6XWYKDeKSukbj9wYEbhHU1RWXDiRMH7dco3zSB1iU9BBOXUP1y+1L947Tp/6Se9LrS+8a03kwlvvkEbT+fMd4HBfuWhjJmZavLM45HXP9EF377LRuz8U5dVF6FFvYzXL8r17VWX98M7b3zyngI1byq4v7YhM2MD7n5hH66gMTMx6L4zjM7uOWdtsb8zKm1aqUdB6y5RvKOwNDupVlfJ5p6jq7btTAuNL0jVRco33D3jCMnr9Wc+ubylI51oviNgK3KPpY/sAt+6lci4Y5kONpKi3tdVf9Y3xJ+X42hhzxVbYatyoeyTUVuHWkvsOROQZpSF0XnnKVOn3RKma7rGnaXmilsSo1UI2yzeINczKWAAAgAElEQVTeR+NI7xDXPuCU2Qy2qkI3jX6jF6RG/I5bVPrI32Lc+trcbo+lg+laGlUat7Cf6Yv3jtO5t3TNhhHnpoi79rqWB64RuEVQZ1bUl1tLF/NivDmnQX97s3wdaWevyp2cs9xNhSPnde+oHgfnUnehr88Mbo5Ke6lQqpIYdpnN29v05pwGfevhSUW9znn/xaFWpnbyq6Waj7jEcbGJ0lXA/3Gy+09Vs4/t+EWpmrD3/fzlivZhlKKNOi6XuG8YWovsLFeO/IVxf6ahs9fEu8IYEbhFUFdXXOfOd5YVHh2Ta9+LY/eupQmb/R4Zt7TbY3E003U1TeRXju3S4Zxem1mvrz00Me9yYVOnRC1LWPPrN5dlcEJnZ+i4+v5VsQY6TrVYSxDHSG6ptoL0dFm2t3Xo9qHBs9OUS4sXuMU1yXwtKtTPtLG5Vaf+7vXOv7vmbY2vDHGft2o1ebtE4BZJnyL7uK3dXKDpR7kP6lLvTGau7KrNujrGnDmJkOc49h/jH7lxaMZzcVxv5qxuUkOI77sS8p2Asj/rnPqmWO6CRy9Ym5FqZdXGVDPZWwGpOGpNU4Lnx4xjlGBbTFfTsEHk6sZtZRvdnbZkXeX716W1tQc3ldbaKP9SpEeLF/ooYxauU/2mrvNhoRircWurmrYV19c47tvNWh6kSuAWQZ1ZUX1KDui/c+DjcXYCz/Zf940r27qzzfGmZ6qFEVx+xZ4fm2PI4zUxQqdff3lvjSHP3M+fmy5J2pIjTU0upuAE02F88e/jdLOvv89qLyXDzQF9gKqlubU98+Ts/f7KjOJm66il628tTWHU1uFCXfzO+tMwffZvo0Ovd+bKTUUHPQX7S8YkqFSlBGjD5zZoWBmb6uK64nSup8TdzskFHj/H/+pVHffL4pIr13KgFTcCtwjqzNQR4q5y9536di4fJNTghOKKVlWxJsbsNsDD6fcvzwo1Qq3UIe/3j1pU1PJBWjs6YgnI031zinHza3P1/huGdP5dalNwvhFytSDqiXrw1Mz+hYWmHytXOfza2jv08+emFTXXsV+1O1T7t0X2MZqrZNtaw1cTvjJ9lS64baRemLKyhNJVR/pzd82Rm/s7+vif39R//O0tXXn/eH3lgQllK1Nc+6w/H1tRr8uTHL1UYQfDhVWOLiNxIXCLINVUWni5fl4uhCgn1Rq6ka6IdLC1Niv/09z6zbp7+EJ9M0RH+84Dr8htF8fFL908UkhLGTou3/bGPG32NY/2sl0np9dn1mtNU+6Rjelp1iop+yI+esE6/XPsUv3s6e7pWcIo9gIad3Od/1L3k6fi75oxf02qT+ic1U2xr7tc0ps4XeP9Rp6atIUNW/T20tKn+au0UkaLS+UZYbuwobZaesqJwC2CoAS8fumdMt0PoK0GRxeVQxydkid4NU1PT16uJyYu0wYvf106oGsL6MyTPXorjvul3780K28+ulzC9hNqqEATTt6UNWV4v0p8Jkn68ePvhF62rb1DX3tooi6/Z2yeZUrbGqXu7k9OXNYtL2N6VaUGVLV0c9ha4vbMp9RakHft0i/mkgQL2obp7zI7P2SPuKEq7d64M3BbsXGrlm3I3XpSqXNJ0hC4RVBXII9bWt904BapJqc2D/NxC7vnR4rjAuDfVj99aqp+8NjbkvI3qyzIMS1KlOLcPWKhZuVJV5JLW3tH6OaIDVtaNHxu7hxrUe3Qp7KH+cm/fb3wQjF4fVb4PkDpfWBxnk7qlRwFuWx9s37y1FR9+5HJGY9HrQErNnCL+yNn7/O10ti0c78+VRsM8E9v1HyurjKf+POb+lfAyPryiuebSa8lyvy+j47P/dmLSZxcC+l9KoXALYKwgVtnjVuEIV+lnnOizLQQxrw13YOlKJ8zLfvOuqFpuxq3tuozf31LUvD2yK4x6aqOr/wBXUyQ/pUHJujK+8eXbfh5P1/g9uDoxQWXf7VA7jnnXEUnYo9DmP5FcY2mDCOdPqHUeVZzee7tFUUtH/YTt3c43TV8gbYWOcAl470KvFkxx2nYJQ/YcydJ0q479g297nzC9GnOlh5d3TfHNBkLGrYEzlySBOn9+BfeAKiw/Of3Wu0GVMujfgncIuhTFy4Bb/rC2dru1Li1VTe+PDuzWc+3ioll7hAed5PBiICaokfHL4u83uxTnHPqbC6Vgk/c2d9Fvi5u5RzJK0n3jFgYugZngRf8pkcExn3n6C/HDS/MKLj8fQUGZ9z/1mId9YtXVF8jk3dHkbEXVPk8PWNlY+QO0StLHNRQyLNvr9CNL8/Wra8XGhVcevmLqS0Me0295MQDJQXfYJYiSs34kfvulvF31LiglCAybbc8geysVZv0xyHFzZQzbE5x28W/l9RSvr+kIHCLoK4uf7qE9AU4XePW3uF048uzddfwBd1GtKXlSsga16697x47xbSmlEK1M3EqlKw0u6YvzuDsphKm/Fodcoqfcp+28p3fC91VBj394tTUiL5imjHet+/uoZetpHi2fTxraWxujXyXv9euOxa1fNj32+rVsDYVqBX2H6KXn3JIt+cHDhqckd/PX+NYjlQmcY+yjVLTvOsOqWDpiH12K7BkOGFriNvaO4oK8j5/1xj9dVi8M+V0a0L3/f3QmCWxvMfnTzo4lvWkPTO5uNrrSiJwi6DO8p9s0k+lq8hb2zu0va298/e0n4eoZs71Nu0drqiTfdyZ2sslqJj+x4I+8/Yc/d/iuB7cWcKUX7VyIxnlztwvSlBRF/ds6iXK9wlK/XT5Nm/uC333F7VG+J723i0VsJ1y6LuLel2udxy9YK1eC7gpe7GINBy5AhR/Lr8bX+66ISrH8eIP3PzrX56nQ3w+cTSnxxVMhq2pOuK6l/WjIgbylCOlTHZRy9Hi0X/n3K1J949aFCqFlF85+x1HReAWQZ+Qc5X6a9yCdtjsWqv2DqdP3z5KQ2fnr83avL1Nh1/7kv46LPwULmHituaWtprrw+SUmcwzaKtnfxUJiVHL3oE7rv49UU7o/ritGrnGyvmOO+/QJ/Dx8YvW66hfvKKR88JdAJ5/Z0XJTaU79vVSDsUU/Xzx7+P09YDp2jYVMauEcy6zL1PAt5BrQFEuXd0fwn3OXPvaGTcNK+p9C60vLUy54pq7tJjjqJi8d9m7YNC1YPmG5kh9cstxbu6T4+awsblVv35xpi7/e+4R5UlD4BZBoabSdMTe1+vj5u88n++Q29jcomkrGnX1E1N8y3d/RXp4eTEjksLUuB1z/RB99A+lndjiElTKQmXPdQca2Mctz6oeCNGBP0nOO3bfkl/r3++ixFv+k2o1J93O27XB92Qxs3/kqumd4PVXHb2g+8jrIIfutWtgrea1z07TNc9Mzfva9Pa9rciZNmIfVer7vVCN0LL1zRnzN+dafsm60vNz7bNHcU3HhcRx05Hu0B+1L2t2kL5pW2tG/sa4BH0vZ9w0TP95z5jQ68jOV/nrf88MXO6V6cFdiMJYl2MgXrr8SZ7KLhuBWwSF8rils5/704F0Bgx5jtmgu+6gc1ofKz7NSL6cOX7FThEzbXl8yUunr2jUmwHV1BmbJeAjZ4/aCtMEHZfhcxs0cNDgsqy7MWJG8Aez+pD4g4NiLh2lNi9J1c9CvrnIk3aYeYXTvvXP4GTQ6RuNoAtf0PHs77y+yZfz61/jlhYc8JMO3IpN3uoPHiYuiT5fqP97LnRaOvMPwzJym+WK86LMM7rHTqnms3OPKf3mxS9MjearM1bnPRfENXp8Xn1mEuIP/PJVHf+rcNNEjV4Qfs7gpeubA28opq8InyYp+/WDpwUHaLlucmavzv9ei9ZuyVmBkZSWl2IQuEXQpy7/XKXpvvL+dCClTsMUGLj5mmDDKmZakNHz14YORj59x6i8zy9ZtyV0X6uLbh8V+HnrMppdussODtJ3WKX0zSo25cUTE0sfSVuoZiLuDPQ3vjK7pGmwZqwsPp9dWrXPnSf+5rWKv2e6ZjHstGUdrmsfnlLkjVAcXQh/8Ojb0Vfik71ff/HvXfMmD5vTPQdfruPAf36zEjO+drh4xmq/mGNQWZpzufOSpd+/2UupErW2c/jc7sFX9rUg17nvVzlqvIKcf+vI2AYQFBK0X0jSdc/mvwkPc1NZy+k9ikXgFkGdWd6dIX2q6NcndbKZW9/UGf3/7Olp+tqD3fuQhJE+OPvEktg3tyFFTrady6K1W/SxP74ZacJ05wpf/NMn/o3NLRm1VKVsnT+/OldH/eKV0MtHuW6mp8KZ7l2ss3epDc3x5uK7e/hC/fCx4i/SUXKOnXnk3iW/NqyFDZu1roiaMilzW8d9FC30+m+FrQVzLtzFZU3TNv1zbOaFtNQaze0x55Hz27Q1d83SV/7RfR7OXKexjMCtyLgtvdzkmKaRamnLfzO3sbm1YrXLYbocFErtEySo9BMKpKn6zAkH5F9nyG3SXuJsG/kGO5Q79VM1ELhFsHR9c7e5NIP0qUtt5n+OzbwTe31W/sEHG5pbtcKrGfHfLz42IbWect9BZB9sG0sMINL5vsYGzLJQXIG6fg367Au9fkkn/Po1Hf/rriaD1wNGxxU6lJ+cFD0XXbEW5uhXVY4Tz9aQtYn+zRzlIt+vArM3fPzPw0vudC7F39+rb57PHDy4JlwBvvnwJP38uekZo+RKzYU1vYzzs941vMiR2LkCtxi+mLgSkadH7+ayIkdNdlANd9R5iltzHI+rfLn8huaZF7VzPdlTBYbsqiNJFx+fCtiOO6h/3vcIewbLd8xE1XPq2wjcIhk5L1VVnevOJzsdSFj+pU+/cWjGuiTpiYnLU4+ll6/QDUWh6upc0k2cUQNNfwATtKZc2bu3t3WU1DRYjMCTXbHN4TnXXUKBCkgPlCnmK6m1kcZBwgaknTJq3OI9tadr2ospSpiaiXQiav95p9QJtsvdelTMvpsr+Aw6vzZty9/lwzmnh8cuyegrGMeNbpj5bIM+clB3lo3NrZGOqVzXnQ//fmhXWUJs/zCbJVd3nLhnp1laZMqO7HIEP+n97EGRG4FbDGbm6PuT3aQpxVN7kg4EK91kX+qIpfTHj9KiG/WjlrNJKJeiv58cLyhH7r1Samg6R8MV8dK59U16fWZ9ZpNkDZ1Ac22HOPaXvnW5T69jAjpht3d0hLoApoM7/5IfOGjPossnFbEfFFjuT0Pm6JI7RkW6gOd6pb+JPn0kFBqsMWV5o37x3HT9ccickssTZKd+pV0yz/zDsMBtHSZwc87p31NWdgvUWtqdxi1cp4GDBkcaeRvmhiXXfpI+NzUU6KJQzF7x0JjFgY9vbWnXyHmpQWDlrClOAgK3GOTqfPyMN29gnyIvvP6RVkEavBGfUWoI9tk93mHy+Vhn4BbhpJ712igjzaT4Z3wI+oaLjbdyBbblqHGbW79ZzS2FA3F/kUpJ43HuLSP0tax8YLU0GbS/0/Wy9V21sveMWBh53X3z1LjNWtX9Zi9sH8KgtR6w585hi5Uhrpx6dwybrynLGwOnwAsr1/khI3ALeSxEqcl6edoqXX5PcM6vs4/ap+Drc5UxaFuHOSW+NrNe33/07W6pXjZvb9PTk1OtL7m6ofgrCnI1F6/ZVLhfaM5zk/fz7uH5j5diAvrrnw+elu/7j76tK+4bLyl3cNdbELjFoJiTRJgTz/cendztMf9+31md7D1WynV9TdP2suT8yTZpyYauGoIqXq+nZd2hlZKr7eGxS/T5u0aHXr7Y2tVcwULYPGC55KqNeWfZxqKCqHTG+6iBZC3VuN2SY+7NYrOmNza36s+vzsm4OBfbRWLnHYpLlOzfjunvpNi5iEPfTIX80n9ZxGjFbLmK0lzCxPZRdtFvPzJZY3IEQv6a0qcnLe+8iQ7z7kGfL0wC4o3ejfztQ+erzXfzNGpeQ+f7FxrtKuX+rrNTPwWVPmd/vAr2+x/jS2GyNUfuRL90H9Cz/lhav9cdKtAvt1S1W7IESCc2zdfJ9OlJyzVpaVd+pDDnyZUbi5u8u9Tr4Ed+/0be57PP1bkuZmuacpd30pL1ofq4FbojW9CwJVJNTfAJtji/eG66JiwOznUVdF17KSuZ5NysvEvZcnVsjmpqjtQS1Zr+bHUPmJw+269fnKnbh87X4de+1DlHbbEDMjZl1bSvatyqDwalMQn42tKHT7FzEXdUvgdBTrnOAaXUfgZNsRbH/cJjE1JNtKsbt+nqJ6fomw+HzwwQFDhdelf+JLbz6psyjtP1vgFiG5pbO+e2Tve3zrbSN1AhvZ43s1JudKvpDdi/Rsxt0N3DF3T7jip5DsmYhSOrHEGl+MLdqW27wcswUOz3v9duOxT5isohcIvge2cfKSl1J5TL1U9OKTpoyK7Sds5pcUAfhuwdcWtLuwYOGqz7Ri3K2e/Ozz99zZbtbdq8vS3jji5fjZH/vHjKb3MHgM51LZtvdFgpTTbpRJbhOuAWOVAgz+LpVAwPj1mct69Fdofx+0MOza9UjVSYk27Y7baxuSV0+piHfc2Tl/z1rbyjD5MwIEJSxgjPGStT+4R/hNzvX56VsXzQZs1OGP3ytNUZ54Iw+eBmr27SZO9G8ZpnpnUObsolrj5ucUifAjY2t2Tkj1y9aZvm1Tdp5catWpRnRgv/d/Dc2+WdIDzdbSAoUfmc+uBzbylb8JO3jMg8Cxe5Ev85KL2eq7JSsazYuFWbt7d1jv7P5fcvz+5WQxh0Bplb36RvPDQxUvqgIP4WojCbYVVjtBvEqK8vJwK3CHboW/zmyw5eTvtd/lovKZVl+vtZyTHHLVzXeS5NJ9VN34395sWZuuC2kZqyrHDuovTQ8eN/9aref8MQHXHdy3mXX1jk3IJSVwCY79wfZqaG7AAxneiyn68TeK4A8G9vLtBPnownke3Pn5uuiYvX6xfPz9BFt6cSD4e573xswrKCgdCfhszRBbeNLKlc05Y3xt5sH9QvJugjfPufk/XNhydpTYjaNP/7Tlm2MWOi8Wxn3DS06MmhRy9YWzDvVBjOOT09abm2F8jdJUnjfe/X7N1APTGhqwN9dh+gMLXH2YGaP81Dei1B6/vs31LN+Y+OX6oVG7eqtb1Djc2t+vTto7oFPmEDt5kBffImhtjGq4poPUh/hnQtkt8nbxmhj9w4NO+gBP93PjGgZjxoQIgkdXS4wKDFf6zuv2dXTea21nad6U0JuHxD91pyf1/JXOsrhn+Mi5N0wsH5U29I0hX3jev2WJ2Zlgb0De5TJ3369lE6NcS1aGtLduqQ7sv89KmpenVmvaavLN8AgsFTV4WeAzgt1/Zva+/QwEGD9cBb3W+s2ztc0eefSiBwi6DY4f5S92rpMM1Gc1d3b2L7z3vGdjv5Z5cmzPRW6Wa0sEl8u/LKhfPqzHo9907q7nfGyk05p29KL1OM9GjdfffsGmjx9tLgpsz1W1r05KTlRb9HLtlNHKETTBbYznePKDL3lWd14zZ9+o5RRaVsMRWuSMmXwHNba7tGz1+rSUs2dPa7DJObase+wZOyB1m7uUX3jizcVLbad3f8xb+P0+cLNEGF8erMel395BTd+npxiaPT02VlH9vn3zpCl/z1rdDreSWrBtM56Y1Z9Z21KNl5IXO56LZR+uHjb2vaisZuE8cvKDGNiCS9NK1wDWtzETcSn7x5hLa2tKutxPbb9g6njg6ndv/Ugj7Zg2TSfv78dJ36uzc0Ym6DJvmm/fIfqv7a6Q9kTSsVdgqrfxcx0btfxowxLnNka67uFYFNpyYt39j9mrBhS2vemky/7OPb3ypztJewvDOLQJkSw6ddcd/4rvNpiNPvFl9fyVemr9a5twxXe4fr3Ef/9Gr3/q43vzZHZ/5hWKTp/sqhuN6wyFBKUtFSqo83bw8++WVfdLNPVmGOm4mL1+u8Y/cLXn9AeNbhUjmUwt48TlqyIeNkeMZNQ/XNjx3WbblbX5un75x1RLiVeuoCBj20FsiztHbz9oJJNKX4Rttla25t1x559ptc5V+xcasO7N81cvBjfxymc47et1tg9fay8PNNzli5Se8/MH8aiaCRpOmJwW94YYbmr0nVjvhrJIL496Wg5uX2DpeRNsdvm9cROd+F4LQC/TVLkR7dXWxXh1zTCc0OuAHLli8NiZPTf/tmW8nXdOofZTinvklzvP6V89dszmiGLJQu48LbRurf3ztDQVfGlvbCQdmkImo+N29v05z6ppwTkBfiXKpf08QlG3T0/nt0ez7XuTc9x+WX7x+vC47rOhfeM2KhvvnRw1RXZxnn1uz1hE058vqs/MlwwwQ6xfTzDQrGgs5rv34xc3vni4E+d+dovfKjMwOf29raLudcZyLxufWb9eacBl197nvL1tK+bst27bP7TjnPmxtyjKRNzy+8aO3mzn6hQTVyw2anavUamrbroHftEkeRY0GNWwQ7ltBUur6E2QfuD6jClboPAc/usxSmav7vIxdl1FZkvr77Yys3btVxvww3kXGQpu1tgXc2pWQR//OrczRw0OCMk1GhgCuoaSPIupgyrWcrNYnx6TcO7WxW37y9TUvWNQfWhhVzl5trome/7KYpfxCTDtqkwv1B/PvSqPndawPufDN3P9HWjg41bWstauL3WHhlfmrScv3mxZn65M3DYxlA4pzLGaB98+HgCesl6X8ez2zq98//m32sXpYjnUWxZqzcpMvuGasXp3avLZrmTTKe7zyzpcgRoX8fuTBUgBuk3TlN9G4Sswd6hOWvRbzpldl6ZHx6lprcryllhHqQXPuEvxk3NS1auPV99m9ZtbuucALh8YvWZ/R9DnL+rV1dObIDyUOveamz6861z07THcPmdw4OKIcbvNQh7TlqaQvNUTx7dVNgXsS0oC4CtYDALYJSatzCTjgdhv9Ob+CgwRkXUqlw7VNartqKoBPSNc9MC1/AMkufYPxBw38F9O3w+0wRTVVhzVndpGdDdoZ+KUSwlMvqTds0at7anH11pOJmDihlPNjJv3097/O5RkRnjwx8eMxifeeRriDlT6/O1eMTluqwawZnDJCRUsHocb98VaeE6IPjN+jpqZFqTv1dDe4btUjz1mzWkpBNSvk8OWm5nn+ntGYzvy0t7Trs2pc0bM6a2PMS+o1fvF5NARfzKcs26l/jlmbUqEc1OERai1z856YwAfbAQYP18JjFeZf5xXPTKzY5ea7a8t+91NUHdMbKTRoX8hqSHYDNWNnYmfctl/RIzLDCbJo6K1/uxpenpwLtUueH/d6/3u6slcuXduY//ja623mpmmgqjaBfCTVucco+aL50b2bQcs0zUytYmuiaW9q0UxH9n6K8T5zOu3VE6GXbO5yu+sf4kt+rUGBaHyKZZtq4RetL7neTS64Tf3ZA+YuAJJs/ezp14c0O/p4rMch5bMIyPTZhmUb+9OySXj8ioJ/QTTFk4v/pU/Eel0ETtlfKtc/Wzo1cKYL2w2y3D52vLTGfM4Lc+lrhvpTZfRTzyb5p+eK9+c8d5XLjy7N1yqHvLu+bhIwLh81e020AV3qgSSGPjl+qKz48sMiClYdV6m6i0k466SQ3cWL4nbwUre0dOrLAKEwAAHqiSz90kJ4KMejro+8dEGlWjXwW/O4CDZ+7Rl99oLzXe0lafOOFZV2/mU1yzp1UaLmaaCo1s/PNbI6ZzTezQQHP72hmj3vPjzOzgZUvZXelNJVGtcsO5a+R6s2O2m/3ahcBFXLVRwZWuwidDtt712oXAShamKBNUtmCNkm67Y15NTUbSyVUPXAzsz6S/irpU5KOkXS5mR2Ttdh/S9rgnDtC0i2SbqpsKXP733PfW9H3e+++8QQWr//4Y7Gsp6e57OSDq12ETr/7j+OqXYTYTbnh3FjWM3Cv6CO8vnP24ZHX8e2zoq9Dkk49LJ6mpHOO3jeW9dSKUid1z3ZAgVHPlVRK/s8gnz3xwFjWE4evnn5o1d77L2/M04O+pN73X1Wwwirxqh64STpF0nzn3ELnXIukxyRdkrXMJZIe9H5/StInrNTeiDH73seP1LD/PUuLb7xQ111wtEb85Gz96+un6olvfrjbsoXu8G+//ER98pj8J94rTntPyX12/I7YZzdNuO6cSOuY9stztfjGCyNXH9/0udoJUOLqwzD4B2dk/F1KoHHaYe/W7ZefGLks0391nq7+ZLQbjEtOOEBTro8edO25c3Fzaeby5k/O1k/Pf1+kdeyz+076w6UfKPn1z37nI/rZ+UdlbJfHvnFaSes67sD++ud/n1pyWdJu+c/jdeaRe0daxzlH76u7r/hQ5LIs+v0Fkdcx7tpzNOIn0c93w396dixB7bRfRj8GvnvWEdqvyKnJgvz5C8dHXsdfLjsh8jok6fpPZ9e1FG+f3XfUyz8MTjVSiL9G7+NH7VvycZj2yWP21YmHZCY6jnpcxanqfdzM7FJJ5zvnvub9fYWkU51z3/MtM91bZrn39wJvmeAJ2lSZPm6FdHQ41dWZtnqjVXbeoY+2t7VrXv1mtbR3aP89d9LyDVu1U98+Oi5rIvBtre1yLvWaOaubtHxDs1Y1btN/nfYeSdLPnpqqxyemUjWM+MnZuv+tRXr27RXaZ/cd9ecvHK+L73hLfeqss4Pq4hsvVHNLm465foie/NaHdfLA1B1+/aZt+vY/J2ny0o2698sn6eonp+izHzxQ115wtC69a4z22KmvDh+wm/73vPdp1x36aE59k9Y2tejwfXbV/nvunFHmNZu26aExS7Trjn31wUP669C9d9XQ2Wt08qHv1m1vzNO5x+ynDuc6Z4E4fMCuev57Z2i3Hftq6Ox6LVizRb99KXNqoJ+c9z79a9xS3fDpY3TCwf319Ycn6a7/+qB27NtHW1vbdcCeO+nQa17S1888VNddeIyemLBMR++/h449YA81eYkx/+Ovb+nm/zxBb81fmzES9ztnHa6fnn+UJOlL947VZScfok8ff4CkVKLTb/1zkj723n30+qx6fffsw3Vg/11094gF+sBB/dW4tVU//MQROn4Lh9IAABEYSURBVLD/Lt1G5U65/lztuUs/rdm0TXPqm3TmkQM6n/ufx9/JGIF63rH7asiMeh02YFf137mfJi9N5Uj7xFH76L6rTu5cbtaqTfrUX7rPqLDfHjtp9aZt+swJB+gzJx6oEXPXatT8Bp155AB9+vgDdGD/nTVg91TeuvSxvnzDVrV3OJ31pzc71/Pi98/QsQfsoXGL1qtfH1PTtjadeeQAPTRmsX7175ma99tPqV+fOq3bvF0f+r/XddVHBmaMOs7+22/sNZ/Q0NlrdNHx+2uPnfppy/Y2HXvDkMBld9+xb+f39n+feX+3aaDGX/sJ7bFzP+3UL9VlYHtbu+au3qwv3D1GW1vb1bfOciaTvu/KkzRq/lrtskMfHXfgnjr//ft3W+a0372hb3z0MJ33/v10+o1DdfzB/TtnIHn9xx/TOTcPlyS99IMzdcwBXbnCVmzcqnfvsoN29royzFy5SSs2btWZR+6t2aubdOmdozPK9aVTD9GXPzxQo+av1cqNW/WLi1IXvoam7Vq+oVm779RP59w8XD+/8Gg9M3lFRlqCL516iE4a+C79z+NTdMQ+u+m2y07UhMXr9eHD9+qskX960nJd7Zsp5MLj9td7991dt7zelYrnz58/Xu/bb3dddPsovfSDM3XEPrvpiYnLdNnJB6tvnzptbWnX1U++o7Pft49+4g2o+MdVJ2uHvnX6yOF76X2/eCUjp9kBe+6klY3b9MBXTtZZ79un8/GZKzfpyn+M1+Dvn6F99thJLW0d6nBOO/Xro1Hz1qp+0zZ96rj9tHzDVn3twYmdyZyf++7pGbMEzF69Seu3tOgjh++ttvYOzVi5Se3O6a15a7Vo3RY9Mzl1XL1rl37a0Nyq33zm/Rqw2w46sP8uGefXtvYOzVy1SW/NX6ebXuk+a8cOfev0g48foftGLdLxB/fXm3NSQUH2Deo/3lqkxycs0+zVTXrtfz6qwwbspj+8Mlt3+0ZQv/C90/XOso16776767/uHae2DqcFv7ugM2fhyHkN+tlTU/WevXbNmNj+guP208XHH6CfPzddaze3dH7vI+Y1aNn6rZpw3TkasPuO2rK9TVu2t2no7DUa9Mw0fenUQ3TQu3bRTa/MzjiWfnXxsbrhha6BGKcd9m498JVTtFO/Ppq5clO32Vp27tdHt11+oo7ab3c9On6pPn7UPjpp4Ls1f83mzmMgLX0e7ehw+vnz03XdBUeroWm7trW1646h8/Wib6Twbz7zfjnndP3zM/Snzx+vTx6zr9raO/TazHpddsohklJzX4+cu1ZH7LNbzoTV3z37cB17wJ76ziOTuz3n/56emrRc//vkFJ1/7H56debqbrlN37PXLloSMJtE+jv6+4iFmrGyURd+4ACdeeTeneedcgnbx61HBW5m9g1J35CkQw455ENLliwRkM05V/Lw8bjMXr1JO/fro/fsldm3qb3DqbW9o+wniLik84gFTeod9vWlvrY3qoV9F+Wzva29qJlFkPLK9FV69647ln/0apmFDdxqIR3ICkn+jkUHeY8FLbPczPpK2lNSt2RWzrl7JN0jpWrcylJaJF4tXPiO2q97ZncpNY1Xn7rknLijBl0EbcWphX0X5UPQVpqg2vOerBb6uE2QdKSZHWpmO0i6TNILWcu8IOlK7/dLJQ111a4qBAAAqLCq17g559rM7HuShkjqI+l+59wMM/u1pInOuRck3SfpYTObL2m9UsEdAABAr1L1wE2SnHMvSXop67Hrfb9vk/T5SpcLAACgltRCUykAAABCIHADAABICAI3AACAhCBwAwAASAgCNwAAgIQgcAMAAEgIAjcAAICEIHADAABICAI3AACAhCBwAwAASAgCNwAAgIQgcAMAAEgIc85VuwxlYWYNkpZU4K32lrS2Au+TNGyXYGyXYGyX3Ng2wdguwdguwZKwXd7jnBtQaKEeG7hViplNdM6dVO1y1Bq2SzC2SzC2S25sm2Bsl2Bsl2A9abvQVAoAAJAQBG4AAAAJQeAW3T3VLkCNYrsEY7sEY7vkxrYJxnYJxnYJ1mO2C33cAAAAEoIaNwAAgIQgcCuRmZ1vZnPMbL6ZDap2ecrNzA42s2FmNtPMZpjZD73H321mr5nZPO/nu7zHzcxu87bPVDP7oG9dV3rLzzOzK6v1meJkZn3M7G0ze9H7+1AzG+d9/sfNbAfv8R29v+d7zw/0reMa7/E5ZnZedT5JvMysv5k9ZWazzWyWmX2YfUYys//xjqPpZvaome3UG/cZM7vfzNaY2XTfY7HtH2b2ITOb5r3mNjOzyn7C0uTYLn/0jqOpZvasmfX3PRe4H+S6TuXa15IgaNv4nrvazJyZ7e393TP3Gecc/4r8J6mPpAWSDpO0g6Qpko6pdrnK/Jn3l/RB7/fdJc2VdIykP0ga5D0+SNJN3u8XSHpZkkk6TdI47/F3S1ro/XyX9/u7qv35Ytg+P5b0L0kven8/Ieky7/e7JH3b+/07ku7yfr9M0uPe78d4+9GOkg719q8+1f5cMWyXByV9zft9B0n9e/s+I+lASYsk7ezbV67qjfuMpI9K+qCk6b7HYts/JI33ljXvtZ+q9meOsF3OldTX+/0m33YJ3A+U5zqVa19Lwr+gbeM9frCkIUrlb927J+8z1LiV5hRJ851zC51zLZIek3RJlctUVs65Vc65yd7vTZJmKXUBukSpi7O8n5/xfr9E0kMuZayk/ma2v6TzJL3mnFvvnNsg6TVJ51fwo8TOzA6SdKGke72/TdLHJT3lLZK9XdLb6ylJn/CWv0TSY8657c65RZLmK7WfJZaZ7anUSfY+SXLOtTjnNop9RpL6StrZzPpK2kXSKvXCfcY5N0LS+qyHY9k/vOf2cM6Ndakr8kO+ddW0oO3inHvVOdfm/TlW0kHe77n2g8DrVIHzU83Lsc9I0i2SfirJ33G/R+4zBG6lOVDSMt/fy73HegWvqeZESeMk7eucW+U9tVrSvt7vubZRT9x2typ1wujw/t5L0kbfSdb/GTs/v/d8o7d8T9wuh0pqkPQPSzUj32tmu6qX7zPOuRWS/iRpqVIBW6OkSWKfSYtr/zjQ+z378Z7gq0rVBknFb5d856dEMrNLJK1wzk3JeqpH7jMEbiiKme0m6WlJP3LObfI/592h9KphymZ2kaQ1zrlJ1S5LDeqrVJPGnc65EyVtUarpq1Mv3WfepVRNwKGSDpC0q5Jfg1gWvXH/KMTMrpPUJumRapelFpjZLpKulXR9tctSKQRupVmhVHt62kHeYz2amfVTKmh7xDn3jPdwvVe9LO/nGu/xXNuop2270yVdbGaLlWqK+LikvyhVJd/XW8b/GTs/v/f8npLWqedtFyl1t7rcOTfO+/sppQK53r7PnCNpkXOuwTnXKukZpfYj9pmUuPaPFepqTvQ/nlhmdpWkiyR9yQtqpeK3yzrl3teS6HClboKmeOfhgyRNNrP91EP3GQK30kyQdKQ3MmcHpToMv1DlMpWV1y/iPkmznHM3+556QVJ6RM6Vkp73Pf5lb1TPaZIaveaPIZLONbN3eTUP53qPJZJz7hrn3EHOuYFK7QdDnXNfkjRM0qXeYtnbJb29LvWWd97jl1lqBOGhko5UqpNsYjnnVktaZmbv8x76hKSZ6uX7jFJNpKeZ2S7ecZXeLr1+n/HEsn94z20ys9O87fxl37oSx8zOV6pLxsXOuWbfU7n2g8DrlLfv5NrXEsc5N805t49zbqB3Hl6u1EC61eqp+0y5Rz/01H9KjVaZq9SoneuqXZ4KfN4zlGqymCrpHe/fBUr1l3hD0jxJr0t6t7e8Sfqrt32mSTrJt66vKtWBdr6kr1T7s8W4jc5S16jSw5Q6ec6X9KSkHb3Hd/L+nu89f5jv9dd522uOanAkU4nb5ARJE7395jmlRnD1+n1G0q8kzZY0XdLDSo0I7HX7jKRHlern16rUBfe/49w/JJ3kbeMFku6Ql3S+1v/l2C7zleqXlT7/3lVoP1CO61SufS0J/4K2Tdbzi9U1qrRH7jPMnAAAAJAQNJUCAAAkBIEbAABAQhC4AQAAJASBGwAAQEIQuAEAACQEgRuAmmVmi83Mhfh3VrXLGoaZ/dIr7y+rXRYAydS38CIAUHVDlJq3Mpd8zwFAj0HgBiAJbnTOvVntQgBAtdFUCgAAkBAEbgB6DDMb6PUhW2xmfc1skJnNMrNtZlZvZg+a2SF5Xn+smT1kZsvMbLuZrTWzl8zsUwXe9zwze8bMVppZi5mtNrO3zOxnZrZzjtfsa2Z3m9ly770WmdmNZrZTwLJ9zOxbZjbazBq996g3s8lm9mczG1D81gKQRARuAHqqx5WaE3SpUvOkbldq0ugJvonvO5nZxZImSbpCUqOkp5Wa/P08SS+Z2W8CXmNmdqekVyT9h6QV3uumSDpY0o2S9g0o28Hee10kaYykNyXtI+lnkp4IWP4+SXcqNffrOElPee+xp6QfSzq8wLYA0EPQxw1AT/QeSTtLOtE5N1OSzGwHpQKg/1JqYvdT0gub2X7qmuz9aufczb7nzpI0WNLPzWyUc26I731+KOlbkuolfcY5N9b3OpN0tqQNAeX7qqR7JX3XOdfiLX+0UhN/f9rMTnfOveU9/h5JVyo1wfjJzrl6/4rM7ARJK4vaOgASixo3AEkwLE8qkI05XvObdNAmSV6A9H1JmySdbGan+5b9uqQ9JL3lD9q8170p6Xbvz/9NP25mfSVd5/15lT9o817nnHNDnXONAWVbJukH6aDNW36WUsGjJH3Ct+w+3s/J2UGb97p3nHNrAt4DQA9EjRuAJMiXDqQ5x+P/zH7AObfRzP4t6UuSzpL0lvfUx7yfD+ZY1/1KNWOeYWZ9nHPtkk6StLek5c65Vwp+gkxDnXNbAx6f7f08IOuxJkkXmtm1kh5xzi0p8v0A9BAEbgCSoNh0IBudc7lq4hZ7Pw/yPXag93NRntd0SNpJ0l6S1ijVHCtJc4ooV9rSHI9v8n52DlBwzjWZ2VeVCh5/K+m3ZrZCqb5xgyU95pzbVkIZACQQTaUA0MWVadlsHcUs7Jx7StIhkq5SKoDbLOlSSf+QNNvMDo5QFgAJQuAGoCfqb2Z75nhuoPdzhe+x9O+H5XlNnaRtktZ7j6VrzbqNUC0H59xG59yDzrn/ds4dJekIScOUqvm7qRJlAFB9BG74//bu0DWLOI7j+PsT9A8wC5tNLTKUBVFBi0WrBlEUZjXP4JJgGjIWDRYR64rBIAoLC04WjApqMsw0MKjwNfxOeBibToeO3/Z+lXvu7rnfXXr4PL+77/ek3erK+g1DmLswrL4Y2fVyWF7bZKwbw3Kxqr4Pn5eBVeBgkvPbu9Q/V1XvaLdOAY797/NL2hkGN0m71czQYgOAJPuAOVrvs+WqWhz57gNaAcCpJLdGB0lyhlaNCjD7c3tVfQPuDasPk0yuOy5Jzv5i5m9LkkwkubxJI9+Lw9JiBWmPsDhBUg+mk1z/xf7HVfVsZP0jbUZsJclzWkPdk7TGt6usm1mrqk9JrtKa9s4lmQLe0Ko7T9P+5N7doHr0PnAEmAKWkrwC3gIHgKPD+Q4N5/9bY8AT4EuS17RWIvuBCdqt3TVgZhvjS+qIwU1SD353K3IFGA1uBVwCpmlvQhijVWw+Au5U1fv1A1TVQpITtLYf52gP/68N485X1dMNjingZpIFWiPeSdrbDT7TAtw8m7cx2aol4DatZclh4DjwlRbgZodrc8ZN2iPSfnckqX9JxmktPT5U1fiOXowk/QM+4yZJktQJg5skSVInDG6SJEmd8Bk3SZKkTjjjJkmS1AmDmyRJUicMbpIkSZ0wuEmSJHXC4CZJktQJg5skSVInfgDGoxjCMwLgwQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(10,7))\n",
    "plt.plot(losses)\n",
    "plt.xlabel(\"Epochs\",fontsize=22)\n",
    "plt.ylabel(\"Loss\",fontsize=22)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Games played: 1000, # of wins: 942\n",
      "Win percentage: 94.19999999999999%\n"
     ]
    }
   ],
   "source": [
    "max_games = 1000\n",
    "wins = 0\n",
    "for i in range(max_games):\n",
    "    win = test_model(model, mode='random', display=False)\n",
    "    if win:\n",
    "        wins += 1\n",
    "win_perc = float(wins) / float(max_games)\n",
    "print(\"Games played: {0}, # of wins: {1}\".format(max_games,wins))\n",
    "print(\"Win percentage: {}%\".format(100.0*win_perc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeprl]",
   "language": "python",
   "name": "conda-env-deeprl-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
